Merge branch 'chrysn/for-29033' into chrysn/ota

Change-Id: Ie0d9b0c04fe618d3809e117823c519b28d15ea05

.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+.*.swp

.gitreview

@@ -0,0 +1,3 @@
+[gerrit]
+host=gerrit.osmocom.org
+project=pysim

83a7bd6df8d8817aeb6395bf9c0d022d505d3afd

@@ -1,51 +0,0 @@
-{
-  "comments": [
-    {
-      "unresolved": true,
-      "key": {
-        "uuid": "6004c8f9_8be9b3f4",
-        "filename": "pySim/global_platform/__init__.py",
-        "patchSetId": 1
-      },
-      "lineNbr": 588,
-      "author": {
-        "id": 1000004
-      },
-      "writtenOn": "2026-03-16T00:25:18Z",
-      "side": 1,
-      "message": "technically correct, but inside pysim we\u0027d usually use our own h2b() function.   They date back to python2 where there was no \u0027bytes\u0027 type, and where we had to roll our own.  We can clean up the legacy at some point, but until then the code is more readable if it uses our b2h/h2b consistently.",
-      "range": {
-        "startLine": 588,
-        "startChar": 0,
-        "endLine": 588,
-        "endChar": 2
-      },
-      "revId": "83a7bd6df8d8817aeb6395bf9c0d022d505d3afd",
-      "serverId": "035e6965-6537-41bd-912c-053f3cf69326"
-    },
-    {
-      "unresolved": false,
-      "key": {
-        "uuid": "68a28803_dbb1c4e9",
-        "filename": "pySim/global_platform/__init__.py",
-        "patchSetId": 1
-      },
-      "lineNbr": 588,
-      "author": {
-        "id": 1000010
-      },
-      "writtenOn": "2026-03-16T09:56:37Z",
-      "side": 1,
-      "message": "Done",
-      "parentUuid": "6004c8f9_8be9b3f4",
-      "range": {
-        "startLine": 588,
-        "startChar": 0,
-        "endLine": 588,
-        "endChar": 2
-      },
-      "revId": "83a7bd6df8d8817aeb6395bf9c0d022d505d3afd",
-      "serverId": "035e6965-6537-41bd-912c-053f3cf69326"
-    }
-  ]
-}

COPYING

@@ -0,0 +1,339 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

README.md

@@ -0,0 +1,122 @@
+pySim - Read, Write and Browse Programmable SIM/USIM Cards
+====================================================
+
+This repository contains Python programs that can be used
+to read, program (write) and browse certain fields/parameters on so-called programmable
+SIM/USIM cards.
+
+Such SIM/USIM cards are special cards, which - unlike those issued by
+regular commercial operators - come with the kind of keys that allow you
+to write the files/fields that normally only an operator can program.
+
+This is useful particularly if you are running your own cellular
+network, and want to issue your own SIM/USIM cards for that network.
+
+
+Homepage and Manual
+-------------------
+
+Please visit the [official homepage](https://osmocom.org/projects/pysim/wiki) for usage instructions, manual and examples.  The user manual can also be built locally from this source code by ``cd docs && make html latexpdf`` for HTML and PDF format, respectively.
+
+Git Repository
+--------------
+
+You can clone from the official Osmocom  git repository using
+```
+git clone git://git.osmocom.org/pysim.git
+```
+
+There is a cgit interface at <https://git.osmocom.org/pysim>
+
+
+Installation
+------------
+
+Please install the following dependencies:
+
+ - pyscard
+ - serial
+ - pytlv
+ - cmd2 >= 1.3.0 but < 2.0.0
+ - jsonpath-ng
+ - construct
+ - bidict
+ - gsm0338
+
+Example for Debian:
+```
+apt-get install python3-pyscard python3-serial python3-pip python3-yaml python3-termcolor python3-colorlog
+pip3 install -r requirements.txt
+```
+
+After installing all dependencies, the pySim applications ``pySim-read.py``, ``pySim-prog.py`` and ``pySim-shell.py`` may be started directly from the cloned repository.
+
+### Archlinux Package
+
+Archlinux users may install the package ``python-pysim-git``
+[![](https://img.shields.io/aur/version/python-pysim-git)](https://aur.archlinux.org/packages/python-pysim-git)
+from the [Arch User Repository (AUR)](https://aur.archlinux.org).
+The most convenient way is the use of an [AUR Helper](https://wiki.archlinux.org/index.php/AUR_helpers),
+e.g. [yay](https://aur.archlinux.org/packages/yay) or [pacaur](https://aur.archlinux.org/packages/pacaur).
+The following example shows the installation with ``yay``.
+
+```sh
+# Install
+yay -Sy python-pysim-git
+
+# Uninstall
+sudo pacman -Rs python-pysim-git
+```
+
+
+Mailing List
+------------
+
+There is no separate mailing list for this project. However,
+discussions related to pysim-prog are happening on the
+<openbsc@lists.osmocom.org> mailing list, please see
+<https://lists.osmocom.org/mailman/listinfo/openbsc> for subscription
+options and the list archive.
+
+Please observe the [Osmocom Mailing List
+Rules](https://osmocom.org/projects/cellular-infrastructure/wiki/Mailing_List_Rules)
+when posting.
+
+
+Contributing
+------------
+
+Our coding standards are described at
+<https://osmocom.org/projects/cellular-infrastructure/wiki/Coding_standards>
+
+We are using a gerrit-based patch review process explained at
+<https://osmocom.org/projects/cellular-infrastructure/wiki/Gerrit>
+
+
+Documentation
+-------------
+
+The pySim user manual can be built from this very source code by means
+of sphinx (with sphinxcontrib-napoleon and sphinx-argparse).  See the
+Makefile in the 'docs' directory.
+
+A pre-rendered HTML user manual of the current pySim 'git master' is
+available from <https://downloads.osmocom.org/docs/latest/pysim/> and
+a downloadable PDF version is published at
+<https://downloads.osmocom.org/docs/latest/osmopysim-usermanual.pdf>.
+
+A slightly dated video presentation about pySim-shell can be found at
+<https://media.ccc.de/v/osmodevcall-20210409-laforge-pysim-shell>.
+
+
+pySim-shell vs. legacy tools
+----------------------------
+
+While you will find a lot of online resources still describing the use of
+pySim-prog.py and pySim-read.py, those tools are considered legacy by
+now and have by far been superseded by the much more capable
+pySim-shell.  We strongly encourage users to adopt pySim-shell, unless
+they have very specific requirements like batch programming of large
+quantities of cards, which is about the only remaining use case for the
+legacy tools.
+

contrib/jenkins.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+# jenkins build helper script for pysim.  This is how we build on jenkins.osmocom.org
+#
+# environment variables:
+# * WITH_MANUALS: build manual PDFs if set to "1"
+# * PUBLISH: upload manuals after building if set to "1" (ignored without WITH_MANUALS = "1")
+#
+
+set -e
+
+if [ ! -d "./pysim-testdata/" ] ; then
+	echo "###############################################"
+	echo "Please call from pySim-prog top directory"
+	echo "###############################################"
+	exit 1
+fi
+
+virtualenv -p python3 venv --system-site-packages
+. venv/bin/activate
+pip install pytlv
+pip install 'pyyaml>=5.1'
+pip install cmd2==1.5
+pip install jsonpath-ng
+pip install construct
+pip install bidict
+pip install gsm0338
+pip install termcolor
+pip install colorlog
+pip install pycryptodome
+# we need this direct git install, as pypi only lists the python2.7 only release 0.3 from 2013 :(
+pip install git+https://github.com/hologram-io/smpp.pdu
+
+# Execute automatically discovered unit tests first
+python -m unittest discover -v -s tests/
+
+# Run pylint to find potential errors
+# Ignore E1102: not-callable
+#   pySim/filesystem.py: E1102: method is not callable (not-callable)
+# Ignore E0401: import-error
+#   pySim/utils.py:276: E0401: Unable to import 'Crypto.Cipher' (import-error)
+#   pySim/utils.py:277: E0401: Unable to import 'Crypto.Util.strxor' (import-error)
+pip install pylint
+python -m pylint --errors-only \
+	--disable E1102 \
+	--disable E0401 \
+	--enable W0301 \
+	pySim *.py
+
+# attempt to build documentation
+pip install sphinx
+pip install sphinxcontrib-napoleon
+pip3 install -e 'git+https://github.com/osmocom/sphinx-argparse@master#egg=sphinx-argparse'
+(cd docs && make html latexpdf)
+
+if [ "$WITH_MANUALS" = "1" ] && [ "$PUBLISH" = "1" ]; then
+	make -C "docs" publish publish-html
+fi
+
+# run the test with physical cards
+cd pysim-testdata
+../tests/pysim-test.sh

contrib/sim-rest-client.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+#
+# sim-rest-client.py: client program to test the sim-rest-server.py
+#
+# this will generate authentication tuples just like a HLR / HSS
+# and will then send the related challenge to the REST interface
+# of sim-rest-server.py
+#
+# sim-rest-server.py will then contact the SIM card to perform the
+# authentication (just like a 3GPP RAN), and return the results via
+# the REST to sim-rest-client.py.
+#
+# (C) 2021 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import Optional, Dict
+
+import sys
+import argparse
+import secrets
+import requests
+
+from CryptoMobile.Milenage import Milenage
+from CryptoMobile.utils import xor_buf
+
+def unpack48(x:bytes) -> int:
+    """Decode a big-endian 48bit number from binary to integer."""
+    return int.from_bytes(x, byteorder='big')
+
+def pack48(x:int) -> bytes:
+    """Encode a big-endian 48bit number from integer to binary."""
+    return x.to_bytes(48 // 8, byteorder='big')
+
+def milenage_generate(opc:bytes, amf:bytes, k:bytes, sqn:bytes, rand:bytes) -> Dict[str, bytes]:
+    """Generate an MILENAGE Authentication Tuple."""
+    m = Milenage(None)
+    m.set_opc(opc)
+    mac_a = m.f1(k, rand, sqn, amf)
+    res, ck, ik, ak = m.f2345(k, rand)
+
+    # AUTN = (SQN ^ AK) || AMF || MAC
+    sqn_ak = xor_buf(sqn, ak)
+    autn = b''.join([sqn_ak, amf, mac_a])
+
+    return {'res': res, 'ck': ck, 'ik': ik, 'autn': autn}
+
+def milenage_auts(opc:bytes, k:bytes, rand:bytes, auts:bytes) -> Optional[bytes]:
+    """Validate AUTS. If successful, returns SQN_MS"""
+    amf = b'\x00\x00' # TS 33.102 Section 6.3.3
+    m = Milenage(None)
+    m.set_opc(opc)
+    ak = m.f5star(k, rand)
+
+    sqn_ak = auts[:6]
+    sqn = xor_buf(sqn_ak, ak[:6])
+
+    mac_s = m.f1star(k, rand, sqn, amf)
+    if mac_s == auts[6:14]:
+        return sqn
+    else:
+        return False
+
+
+def build_url(suffix:str, base_path="/sim-auth-api/v1") -> str:
+    """Build an URL from global server_host, server_port, BASE_PATH and suffix."""
+    return "http://%s:%u%s%s" % (server_host, server_port, base_path, suffix)
+
+
+def rest_post(suffix:str, js:Optional[dict] = None):
+    """Perform a RESTful POST."""
+    url = build_url(suffix)
+    if verbose:
+        print("POST %s (%s)" % (url, str(js)))
+    resp = requests.post(url, json=js)
+    if verbose:
+        print("-> %s" % (resp))
+    if not resp.ok:
+        print("POST failed")
+    return resp
+
+def rest_get(suffix:str, base_path=None):
+    """Perform a RESTful GET."""
+    url = build_url(suffix, base_path)
+    if verbose:
+        print("GET %s" % url)
+    resp = requests.get(url)
+    if verbose:
+        print("-> %s" % (resp))
+    if not resp.ok:
+        print("GET failed")
+    return resp
+
+
+def main_info(args):
+    resp = rest_get('/slot/%u' % args.slot_nr, base_path="/sim-info-api/v1")
+    if not resp.ok:
+        print("<- ERROR %u: %s" % (resp.status_code, resp.text))
+        sys.exit(1)
+    resp_json = resp.json()
+    print("<- %s" % resp_json)
+
+
+def main_auth(args):
+    #opc = bytes.fromhex('767A662ACF4587EB0C450C6A95540A04')
+    #k = bytes.fromhex('876B2D8D403EE96755BEF3E0A1857EBE')
+    opc = bytes.fromhex(args.opc)
+    k = bytes.fromhex(args.key)
+    amf = bytes.fromhex(args.amf)
+    sqn = bytes.fromhex(args.sqn)
+
+    for i in range(args.count):
+        rand = secrets.token_bytes(16)
+        t = milenage_generate(opc=opc, amf=amf, k=k, sqn=sqn, rand=rand)
+
+        req_json = {'rand': rand.hex(), 'autn': t['autn'].hex()}
+        print("-> %s" % req_json)
+        resp = rest_post('/slot/%u' % args.slot_nr, req_json)
+        if not resp.ok:
+            print("<- ERROR %u: %s" % (resp.status_code, resp.text))
+            break
+        resp_json = resp.json()
+        print("<- %s" % resp_json)
+        if 'synchronisation_failure' in resp_json:
+            auts = bytes.fromhex(resp_json['synchronisation_failure']['auts'])
+            sqn_ms = milenage_auts(opc, k, rand, auts)
+            if sqn_ms is not False:
+                print("SQN_MS = %s" % sqn_ms.hex())
+                sqn_ms_int = unpack48(sqn_ms)
+                # we assume an IND bit-length of 5 here
+                sqn = pack48(sqn_ms_int + (1 << 5))
+            else:
+                raise RuntimeError("AUTS auth failure during re-sync?!?")
+        elif 'successful_3g_authentication' in resp_json:
+            auth_res = resp_json['successful_3g_authentication']
+            assert bytes.fromhex(auth_res['res']) == t['res']
+            assert bytes.fromhex(auth_res['ck']) == t['ck']
+            assert bytes.fromhex(auth_res['ik']) == t['ik']
+            # we assume an IND bit-length of 5 here
+            sqn = pack48(unpack48(sqn) + (1 << 5))
+        else:
+            raise RuntimeError("Auth failure")
+
+
+def main(argv):
+    global server_port, server_host, verbose
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-H", "--host", help="Host to connect to", default="localhost")
+    parser.add_argument("-p", "--port", help="TCP port to connect to", default=8000)
+    parser.add_argument("-v", "--verbose", help="increase output verbosity", action='count', default=0)
+    parser.add_argument("-n", "--slot-nr", help="SIM slot number", type=int, default=0)
+    subp = parser.add_subparsers()
+
+    auth_p = subp.add_parser('auth', help='UMTS AKA Authentication')
+    auth_p.add_argument("-c", "--count", help="Auth count", type=int, default=10)
+    auth_p.add_argument("-k", "--key", help="Secret key K (hex)", type=str, required=True)
+    auth_p.add_argument("-o", "--opc", help="Secret OPc (hex)", type=str, required=True)
+    auth_p.add_argument("-a", "--amf", help="AMF Field (hex)", type=str, default="0000")
+    auth_p.add_argument("-s", "--sqn", help="SQN Field (hex)", type=str, default="000000000000")
+    auth_p.set_defaults(func=main_auth)
+
+    info_p = subp.add_parser('info', help='Information about the Card')
+    info_p.set_defaults(func=main_info)
+
+    args = parser.parse_args()
+    server_host = args.host
+    server_port = args.port
+    verbose = args.verbose
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main(sys.argv)

contrib/sim-rest-server.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+# RESTful HTTP service for performing authentication against USIM cards
+#
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import sys
+import argparse
+
+from klein import Klein
+
+from pySim.transport import ApduTracer
+from pySim.transport.pcsc import PcscSimLink
+from pySim.commands import SimCardCommands
+from pySim.cards import UsimCard
+from pySim.exceptions import *
+
+class ApduPrintTracer(ApduTracer):
+    def trace_response(self, cmd, sw, resp):
+        #print("CMD: %s -> RSP: %s %s" % (cmd, sw, resp))
+        pass
+
+def connect_to_card(slot_nr:int):
+    tp = PcscSimLink(slot_nr, apdu_tracer=ApduPrintTracer())
+    tp.connect()
+
+    scc = SimCardCommands(tp)
+    card = UsimCard(scc)
+
+    # this should be part of UsimCard, but FairewavesSIM breaks with that :/
+    scc.cla_byte = "00"
+    scc.sel_ctrl = "0004"
+
+    card.read_aids()
+    card.select_adf_by_aid(adf='usim')
+
+    return tp, scc, card
+
+class ApiError:
+    def __init__(self, msg:str, sw=None):
+        self.msg = msg
+        self.sw = sw
+
+    def __str__(self):
+        d = {'error': {'message':self.msg}}
+        if self.sw:
+            d['error']['status_word'] = self.sw
+        return json.dumps(d)
+
+
+def set_headers(request):
+    request.setHeader('Content-Type', 'application/json')
+
+class SimRestServer:
+    app = Klein()
+
+    @app.handle_errors(NoCardError)
+    def no_card_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(410)
+        return str(ApiError("No SIM card inserted in slot"))
+
+    @app.handle_errors(ReaderError)
+    def reader_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(404)
+        return str(ApiError("Reader Error: Specified SIM Slot doesn't exist"))
+
+    @app.handle_errors(ProtocolError)
+    def protocol_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(500)
+        return str(ApiError("Protocol Error: %s" % failure.value))
+
+    @app.handle_errors(SwMatchError)
+    def sw_match_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(500)
+        sw = failure.value.sw_actual
+        if sw == '9862':
+            return str(ApiError("Card Authentication Error - Incorrect MAC", sw))
+        elif sw == '6982':
+            return str(ApiError("Security Status not satisfied - Card PIN enabled?", sw))
+        else:
+            return str(ApiError("Card Communication Error %s" % failure.value), sw)
+
+
+    @app.route('/sim-auth-api/v1/slot/<int:slot>')
+    def auth(self, request, slot):
+        """REST API endpoint for performing authentication against a USIM.
+           Expects a JSON body containing RAND and AUTN.
+           Returns a JSON body containing RES, CK, IK and Kc."""
+        try:
+            # there are two hex-string JSON parameters in the body: rand and autn
+            content = json.loads(request.content.read())
+            rand = content['rand']
+            autn = content['autn']
+        except:
+            set_headers(request)
+            request.setResponseCode(400)
+            return str(ApiError("Malformed Request"))
+
+        tp, scc, card = connect_to_card(slot)
+
+        card.select_adf_by_aid(adf='usim')
+        res, sw = scc.authenticate(rand, autn)
+
+        tp.disconnect()
+
+        set_headers(request)
+        return json.dumps(res, indent=4)
+
+    @app.route('/sim-info-api/v1/slot/<int:slot>')
+    def info(self, request, slot):
+        """REST API endpoint for obtaining information about an USIM.
+        Expects empty body in request.
+        Returns a JSON body containing ICCID, IMSI."""
+
+        tp, scc, card = connect_to_card(slot)
+
+        card.select_adf_by_aid(adf='usim')
+        iccid, sw = card.read_iccid()
+        imsi, sw = card.read_imsi()
+        res = {"imsi": imsi, "iccid": iccid }
+
+        tp.disconnect()
+
+        set_headers(request)
+        return json.dumps(res, indent=4)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-H", "--host", help="Host/IP to bind HTTP to", default="localhost")
+    parser.add_argument("-p", "--port", help="TCP port to bind HTTP to", default=8000)
+    #parser.add_argument("-v", "--verbose", help="increase output verbosity", action='count', default=0)
+
+    args = parser.parse_args()
+
+    srr = SimRestServer()
+    srr.app.run(args.host, args.port)
+
+if __name__ == "__main__":
+    main(sys.argv)

contrib/sim-rest-server.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Osmocom SIM REST server
+
+[Service]
+Type=simple
+# we listen to 0.0.0.0, allowing remote, unauthenticated clients to connect from everywhere!
+ExecStart=/usr/local/src/pysim/contrib/sim-rest-server.py -H 0.0.0.0
+Restart=always
+RestartSec=2
+# this user must be created beforehand; it must have PC/SC access
+User=rest
+
+[Install]
+WantedBy=multi-user.target

csv-format

@@ -0,0 +1,16 @@
+This file aims to describe the format of the CSV file pySim uses.
+
+The first line contains the fieldnames which will be used by pySim. This
+avoids having a specific order.
+
+The field names are the following:
+
+iccid: ICCID of the card. Used to identify the cards (with --read-iccid)
+imsi: IMSI of the card
+mcc: Mobile Country Code (optional)
+mnc: Mobile Network Code (optional)
+smsp: MSISDN of the SMSC (optional)
+ki: Ki
+opc: OPc
+acc: Access class of the SIM (optional)
+pin_adm: Admin PIN of the SIM. Needed to reprogram various files

docs/Makefile

@@ -0,0 +1,35 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# for osmo-gsm-manuals
+OSMO_GSM_MANUALS_DIR=$(shell pkg-config osmo-gsm-manuals --variable=osmogsmmanualsdir 2>/dev/null)
+OSMO_REPOSITORY = "pysim"
+UPLOAD_FILES = $(BUILDDIR)/latex/osmopysim-usermanual.pdf
+CLEAN_FILES = $(UPLOAD_FILES)
+
+# Put it first so that "make" without argument is like "make help".
+.PHONY: help
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+$(BUILDDIR)/latex/pysim.pdf: latexpdf
+	@/bin/true
+
+publish-html: html
+	rsync -avz -e "ssh -o 'UserKnownHostsFile=$(OSMO_GSM_MANUALS_DIR)/build/known_hosts' -p 48" $(BUILDDIR)/html/ docs@ftp.osmocom.org:web-files/latest/pysim/
+
+# put this before the catch-all below
+include $(OSMO_GSM_MANUALS_DIR)/build/Makefile.common.inc
+
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%:
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,58 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'osmopysim-usermanual'
+copyright = '2009-2022 by Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle'
+author = 'Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+        "sphinx.ext.autodoc",
+        "sphinxarg.ext",
+        "sphinx.ext.autosectionlabel",
+        "sphinx.ext.napoleon"
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+autoclass_content = 'both'

docs/index.rst

@@ -0,0 +1,50 @@
+.. pysim documentation master file
+
+Welcome to Osmocom pySim
+========================
+
+Introduction
+------------
+
+pySim is a python implementation of various software that helps you with
+managing subscriber identity cards for cellular networks, so-called SIM
+cards.
+
+Many Osmocom (Open Source Mobile Communications) projects relate to operating
+private / custom cellular networks, and provisioning SIM cards for said networks
+is in many cases a requirement to operate such networks.
+
+To make use of most of pySim's features, you will need a `programmable` SIM card,
+i.e. a card where you are the owner/operator and have sufficient credentials (such
+as the `ADM PIN`) in order to write to many if not most of the files on the card.
+
+Such cards are, for example, available from sysmocom, a major contributor to pySim.
+See https://www.sysmocom.de/products/lab/sysmousim/ for more details.
+
+pySim supports classic GSM SIM cards as well as ETSI UICC with 3GPP USIM and ISIM
+applications.  It is easily extensible, so support for additional files, card
+applications, etc. can be added easily by any python developer.  We do encourage you
+to submit your contributions to help this collaborative development project.
+
+pySim consists of several parts:
+
+* a python :ref:`library<pySim library>` containing plenty of objects and methods that can be used for
+  writing custom programs interfacing with SIM cards.
+* the [new] :ref:`interactive pySim-shell command line program<pySim-shell>`
+* the [legacy] :ref:`pySim-prog and pySim-read tools<Legacy tools>`
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Contents:
+
+   shell
+   legacy
+   library
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/legacy.rst

@@ -0,0 +1,100 @@
+Legacy tools
+============
+
+*legacy tools* are the classic ``pySim-prog`` and ``pySim-read`` programs that
+existed long before ``pySim-shell``.
+
+These days, you should primarily use ``pySim-shell`` instead of these
+legacy tools.
+
+pySim-prog
+----------
+
+``pySim-prog`` was the first part of the pySim software suite.  It started as
+a tool to write ICCID, IMSI, MSISDN and Ki to very simplistic SIM cards, and
+was later extended to a variety of other cards.  As the number of features supported
+became no longer bearable to express with command-line arguments, `pySim-shell` was
+created.
+
+Basic use cases can still use `pySim-prog`.
+
+Program customizable SIMs
+~~~~~~~~~~~~~~~~~~~~~~~~~
+Two modes are possible:
+
+  - one where you specify every parameter manually :
+
+``./pySim-prog.py -n 26C3 -c 49 -x 262 -y 42 -i <IMSI> -s <ICCID>``
+
+
+  - one where they are generated from some minimal set :
+
+``./pySim-prog.py -n 26C3 -c 49 -x 262 -y 42 -z <random_string_of_choice> -j <card_num>``
+
+    With <random_string_of_choice> and <card_num>, the soft will generate
+    'predictable' IMSI and ICCID, so make sure you choose them so as not to
+    conflict with anyone. (for eg. your name as <random_string_of_choice> and
+    0 1 2 ... for <card num>).
+
+  You also need to enter some parameters to select the device :
+   -t TYPE : type of card (supersim, magicsim, fakemagicsim or try 'auto')
+   -d DEV  : Serial port device (default /dev/ttyUSB0)
+   -b BAUD : Baudrate (default 9600)
+
+
+pySim-read
+----------
+
+``pySim-read`` allows you to read some data from a SIM card.  It will only some files
+of the card, and will only read files accessible to a normal user (without any special authentication)
+
+These days, you should use the ``export`` command of ``pySim-shell``
+instead. It performs a much more comprehensive export of all of the
+[standard] files that can be found on the card.  To get a human-readable
+decode instead of the raw hex export, you can use ``export --json``.
+
+Specifically, pySim-read will dump the following:
+
+* MF
+
+ * EF.ICCID
+
+* DF.GSM
+
+ * EF,IMSI
+ * EF.GID1
+ * EF.GID2
+ * EF.SMSP
+ * EF.SPN
+ * EF.PLMNsel
+ * EF.PLMNwAcT
+ * EF.OPLMNwAcT
+ * EF.HPLMNAcT
+ * EF.ACC
+ * EF.MSISDN
+ * EF.AD
+ * EF.SST
+
+* ADF.USIM
+
+ * EF.EHPLMN
+ * EF.UST
+ * EF.ePDGId
+ * EF.ePDGSelection
+
+* ADF.ISIM
+
+ * EF.PCSCF
+ * EF.DOMAIN
+ * EF.IMPI
+ * EF.IMPU
+ * EF.UICCIARI
+ * EF.IST
+
+
+pySim-read usage
+~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: pySim-read
+   :func: option_parser

docs/library.rst

@@ -0,0 +1,111 @@
+pySim library
+=============
+
+pySim filesystem abstraction
+----------------------------
+
+.. automodule:: pySim.filesystem
+   :members:
+
+pySim commands abstraction
+--------------------------
+
+.. automodule:: pySim.commands
+   :members:
+
+pySim Transport
+---------------
+
+The pySim.transport classes implement specific ways how to
+communicate with a SIM card.  A "transport" provides ways
+to transceive APDUs with the card.
+
+The most commonly used transport uses the PC/SC interface to
+utilize a variety of smart card interfaces ("readers").
+
+Transport base class
+~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.transport
+   :members:
+
+
+calypso / OsmocomBB transport
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This allows the use of the SIM slot of an OsmocomBB compatible phone with the TI Calypso chipset,
+using the L1CTL interface to talk to the layer1.bin firmware on the phone.
+
+.. automodule:: pySim.transport.calypso
+   :members:
+
+
+AT-command Modem transport
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This transport uses AT commands of a cellular modem in order to get access to the SIM card inserted
+in such a modem.
+
+.. automodule:: pySim.transport.modem_atcmd
+   :members:
+
+
+PC/SC transport
+~~~~~~~~~~~~~~~
+
+PC/SC is the standard API for accessing smart card interfaces
+on all major operating systems, including the MS Windows Family,
+OS X as well as Linux / Unix OSs.
+
+.. automodule:: pySim.transport.pcsc
+   :members:
+
+
+Serial/UART transport
+~~~~~~~~~~~~~~~~~~~~~
+
+This transport implements interfacing smart cards via
+very simplistic UART readers.  These readers basically
+wire together the Rx+Tx pins of a RS232 UART, provide
+a fixed crystal oscillator for clock, and operate the UART
+at 9600 bps.  These readers are sometimes called `Phoenix`.
+
+.. automodule:: pySim.transport.serial
+   :members:
+
+
+pySim construct utilities
+-------------------------
+
+.. automodule:: pySim.construct
+   :members:
+
+pySim TLV utilities
+-------------------
+
+.. automodule:: pySim.tlv
+   :members:
+
+pySim utility functions
+-----------------------
+
+.. automodule:: pySim.utils
+   :members:
+
+pySim exceptions
+----------------
+
+.. automodule:: pySim.exceptions
+   :members:
+
+pySim card_handler
+------------------
+
+.. automodule:: pySim.card_handler
+   :members:
+
+pySim card_key_provider
+-----------------------
+
+.. automodule:: pySim.card_key_provider
+   :members:

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/shell.rst

@@ -0,0 +1,672 @@
+pySim-shell
+===========
+
+pySim-shell is an interactive command line shell for all kind of interactions with SIM cards.
+
+The interactive shell provides command for
+
+* navigating the on-card filesystem hierarchy
+* authenticating with PINs such as ADM1
+* CHV/PIN management (VERIFY, ENABLE, DISABLE, UNBLOCK)
+* decoding of SELECT response (file control parameters)
+* reading and writing of files and records in raw, hex-encoded binary format
+* for some files where related support has been developed:
+
+ * decoded reading (display file data in JSON format)
+ * decoded writing (encode from JSON to binary format, then write)
+
+By means of using the python ``cmd2`` module, various useful features improve usability:
+
+* history of commands (persistent across restarts)
+* output re-direction to files on your computer
+* output piping through external tools like 'grep'
+* tab completion of commands and SELECT-able files/directories
+* interactive help for all commands
+
+Running pySim-shell
+-------------------
+
+pySim-shell has a variety of command line arguments to control
+
+* which transport to use (how to use a reader to talk to the SIM card)
+* whether to automatically verify an ADM pin (and in which format)
+* whether to execute a start-up script
+
+.. argparse::
+   :module: pySim-shell
+   :func: option_parser
+
+
+
+cmd2 basics
+-----------
+
+FIXME
+
+
+
+ISO7816 commands
+----------------
+
+This category of commands relates to commands that originate in the ISO 7861-4 specifications,
+most of them have a 1:1 resemblance in the specification.
+
+select
+~~~~~~
+
+The ``select`` command is used to select a file, either by its FID, AID or by its symbolic name.
+
+Try ``select`` with tab-completion to get a list of all current selectable items:
+
+::
+
+  pySIM-shell (MF)> select
+  ..                2fe2              a0000000871004    EF.ARR            MF
+  2f00              3f00              ADF.ISIM          EF.DIR
+  2f05              7f10              ADF.USIM          EF.ICCID
+  2f06              7f20              DF.GSM            EF.PL
+  2f08              a0000000871002    DF.TELECOM        EF.UMPC
+
+Use ``select`` with a specific FID or name to select the new file.
+
+This will
+
+* output the [JSON decoded, if possible] select response
+* change the prompt to the newly selected file
+* enable any commands specific to the newly-selected file
+
+::
+
+  pySIM-shell (MF)> select ADF.USIM
+  {
+      "file_descriptor": {
+          "file_descriptor_byte": {
+              "shareable": true,
+              "file_type": "df",
+              "structure": "no_info_given"
+          }
+      },
+      "df_name": "A0000000871002FFFFFFFF8907090000",
+      "proprietary_info": {
+          "uicc_characteristics": "71",
+          "available_memory": 101640
+      },
+      "life_cycle_status_int": "operational_activated",
+      "security_attrib_compact": "00",
+      "pin_status_template_do": "90017083010183018183010A83010B"
+  }
+  pySIM-shell (MF/ADF.USIM)>
+
+
+status
+~~~~~~
+
+The ``status`` command [re-]obtains the File Control Template of the
+currently-selected file and print its decoded output.
+
+Example:
+
+::
+
+  pySIM-shell (MF/ADF.ISIM)> status
+  {
+      "file_descriptor": {
+          "file_descriptor_byte": {
+              "shareable": true,
+              "file_type": "df",
+              "structure": "no_info_given"
+          },
+          "record_len": null,
+          "num_of_rec": null
+      },
+      "file_identifier": "ff01",
+      "df_name": "a0000000871004ffffffff8907090000",
+      "proprietary_information": {
+          "uicc_characteristics": "71",
+          "available_memory": 101640
+      },
+      "life_cycle_status_integer": "operational_activated",
+      "security_attrib_compact": "00",
+      "pin_status_template_do": {
+          "ps_do": "70",
+          "key_reference": 11
+      }
+  }
+
+
+change_chv
+~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.change_chv_parser
+
+
+disable_chv
+~~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.disable_chv_parser
+
+
+enable_chv
+~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.enable_chv_parser
+
+
+unblock_chv
+~~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.unblock_chv_parser
+
+
+verify_chv
+~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.verify_chv_parser
+
+deactivate_file
+~~~~~~~~~~~~~~~
+Deactivate the currently selected file.  This used to be called INVALIDATE in TS 11.11.
+
+
+activate_file
+~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.activate_file_parser
+
+open_channel
+~~~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.open_chan_parser
+
+close_channel
+~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.close_chan_parser
+
+
+suspend_uicc
+~~~~~~~~~~~~
+This command allows you to perform the SUSPEND UICC command on the card.  This is a relatively
+recent power-saving addition to the UICC specifications, allowing for suspend/resume while maintaining
+state, as opposed to a full power-off (deactivate) and power-on (activate) of the card.
+
+The pySim command just sends that SUSPEND UICC command and doesn't perform the full related sequence
+including the electrical power down.
+
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.suspend_uicc_parser
+
+
+
+pySim commands
+--------------
+
+Commands in this category are pySim specific; they do not have a 1:1 correspondence to ISO 7816
+or 3GPP commands. Mostly they will operate either only on local (in-memory) state, or execute
+a complex sequence of card-commands.
+
+desc
+~~~~
+Display human readable file description for the currently selected file.
+
+
+dir
+~~~
+.. argparse::
+   :module: pySim-shell
+   :func: PySimCommands.dir_parser
+
+Example:
+::
+
+  pySIM-shell (MF)> dir
+  MF
+  3f00
+   ..          ADF.USIM    DF.SYSTEM   EF.DIR      EF.UMPC
+   ADF.ARA-M   DF.EIRENE   DF.TELECOM  EF.ICCID    MF
+   ADF.ISIM    DF.GSM      EF.ARR      EF.PL
+  14 files
+
+
+export
+~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: PySimCommands.export_parser
+
+Please note that `export` works relative to the current working
+directory, so if you are in `MF`, then the export will contain all known
+files on the card.  However, if you are in `ADF.ISIM`, only files below
+that ADF will be part of the export.
+
+Furthermore, it is strongly advised to first enter the ADM1 pin
+(`verify_adm`) to maximize the chance of having permission to read
+all/most files.
+
+
+tree
+~~~~
+Display a tree of the card filesystem.  It is important to note that this displays a tree
+of files that might potentially exist (based on the card profile).  In order to determine if
+a given file really exists on a given card, you have to try to select that file.
+
+Example:
+::
+
+  pySIM-shell (MF)> tree --help
+  EF.DIR                    2f00 Application Directory
+  EF.ICCID                  2fe2 ICC Identification
+  EF.PL                     2f05 Preferred Languages
+  EF.ARR                    2f06 Access Rule Reference
+  EF.UMPC                   2f08 UICC Maximum Power Consumption
+  DF.TELECOM                7f10 None
+    EF.ADN                  6f3a Abbreviated Dialing Numbers
+  ...
+
+
+
+verify_adm
+~~~~~~~~~~
+Verify the ADM (Administrator) PIN specified as argument.  This is typically needed in order
+to get write/update permissions to most of the files on SIM cards.
+
+Currently only ADM1 is supported.
+
+
+reset
+~~~~~
+Perform card reset and display the card ATR.
+
+intro
+~~~~~
+[Re-]Display the introductory banner
+
+
+equip
+~~~~~
+Equip pySim-shell with a card; particularly useful if the program was
+started before a card was present, or after a card has been replaced by
+the user while pySim-shell was kept running.
+
+bulk_script
+~~~~~~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: PysimApp.bulk_script_parser
+
+
+echo
+~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: PysimApp.echo_parser
+
+
+apdu
+~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: PySimCommands.apdu_cmd_parser
+
+
+
+Linear Fixed EF commands
+------------------------
+
+These commands become enabled only when your currently selected file is of *Linear Fixed EF* type.
+
+read_record
+~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.read_rec_parser
+
+
+read_record_decoded
+~~~~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.read_rec_dec_parser
+
+
+read_records
+~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.read_recs_parser
+
+
+read_records_decoded
+~~~~~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.read_recs_dec_parser
+
+
+update_record
+~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.upd_rec_parser
+
+
+update_record_decoded
+~~~~~~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.upd_rec_dec_parser
+
+
+edit_record_decoded
+~~~~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.edit_rec_dec_parser
+
+This command will read the selected record, decode it to its JSON representation, save
+that JSON to a temporary file on your computer, and launch your configured text editor.
+
+You may then perform whatever modifications to the JSON representation, save + leave your
+text editor.
+
+Afterwards, the modified JSON will be re-encoded to the binary format, and the result written
+back to the record on the SIM card.
+
+This allows for easy interactive modification of records.
+
+
+decode_hex
+~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.dec_hex_parser
+
+
+
+Transparent EF commands
+-----------------------
+
+These commands become enabled only when your currently selected file is of *Transparent EF* type.
+
+
+read_binary
+~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: TransparentEF.ShellCommands.read_bin_parser
+
+
+read_binary_decoded
+~~~~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: TransparentEF.ShellCommands.read_bin_dec_parser
+
+
+update_binary
+~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: TransparentEF.ShellCommands.upd_bin_parser
+
+
+update_binary_decoded
+~~~~~~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: TransparentEF.ShellCommands.upd_bin_dec_parser
+
+In normal operation, update_binary_decoded needs a JSON document representing the entire file contents as
+input.  This can be inconvenient if you want to keep 99% of the content but just toggle one specific
+parameter.   That's where the JSONpath support comes in handy:  You can specify a JSONpath to an element
+inside the document as well as a new value for tat field:
+
+Th below example demonstrates this by modifying the ofm field within EF.AD:
+
+::
+
+  pySIM-shell (MF/ADF.USIM/EF.AD)> read_binary_decoded
+  {
+      "ms_operation_mode": "normal",
+      "specific_facilities": {
+          "ofm": true
+      },
+      "len_of_mnc_in_imsi": 2
+  }
+  pySIM-shell (MF/ADF.USIM/EF.AD)> update_binary_decoded --json-path specific_facilities.ofm false
+  pySIM-shell (MF/ADF.USIM/EF.AD)> read_binary_decoded
+  {
+      "ms_operation_mode": "normal",
+      "specific_facilities": {
+          "ofm": false
+      },
+      "len_of_mnc_in_imsi": 2
+  }
+
+
+edit_binary_decoded
+~~~~~~~~~~~~~~~~~~~
+This command will read the selected binary EF, decode it to its JSON representation, save
+that JSON to a temporary file on your computer, and launch your configured text editor.
+
+You may then perform whatever modifications to the JSON representation, save + leave your
+text editor.
+
+Afterwards, the modified JSON will be re-encoded to the binary format, and the result written
+to the SIM card.
+
+This allows for easy interactive modification of file contents.
+
+
+decode_hex
+~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: TransparentEF.ShellCommands.dec_hex_parser
+
+
+
+BER-TLV EF commands
+-------------------
+
+BER-TLV EFs are files that contain BER-TLV structured data.  Every file can contain any number
+of variable-length IEs (DOs).  The tag within a BER-TLV EF must be unique within the file.
+
+The commands below become enabled only when your currently selected file is of *BER-TLV EF* type.
+
+retrieve_tags
+~~~~~~~~~~~~~
+
+Retrieve a list of all tags present in the currently selected file.
+
+
+retrieve_data
+~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: BerTlvEF.ShellCommands.retrieve_data_parser
+
+
+set_data
+~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: BerTlvEF.ShellCommands.set_data_parser
+
+
+del_data
+~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: BerTlvEF.ShellCommands.del_data_parser
+
+
+
+USIM commands
+-------------
+
+authenticate
+~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.ts_31_102
+   :func: ADF_USIM.AddlShellCommands.authenticate_parser
+
+terminal_profile
+~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.ts_31_102
+   :func: ADF_USIM.AddlShellCommands.term_prof_parser
+
+envelope
+~~~~~~~~
+.. argparse::
+   :module: pySim.ts_31_102
+   :func: ADF_USIM.AddlShellCommands.envelope_parser
+
+envelope_sms
+~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.ts_31_102
+   :func: ADF_USIM.AddlShellCommands.envelope_sms_parser
+
+
+
+
+ARA-M commands
+--------------
+
+The ARA-M commands exist to manage the access rules stored in an ARA-M applet on the card.
+
+ARA-M in the context of SIM cards is primarily used to enable Android UICC Carrier Privileges,
+please see https://source.android.com/devices/tech/config/uicc for more details on the background.
+
+
+aram_get_all
+~~~~~~~~~~~~
+
+Obtain and decode all access rules from the ARA-M applet on the card.
+
+NOTE: if the total size of the access rules exceeds 255 bytes, this command will fail, as
+it doesn't yet implement fragmentation/reassembly on rule retrieval. YMMV
+
+::
+
+  pySIM-shell (MF/ADF.ARA-M)> aram_get_all
+  [
+      {
+          "ResponseAllRefArDO": [
+              {
+                  "RefArDO": [
+                      {
+                          "RefDO": [
+                              {
+                                  "AidRefDO": "ffffffffffff"
+                              },
+                              {
+                                  "DevAppIdRefDO": "e46872f28b350b7e1f140de535c2a8d5804f0be3"
+                              }
+                          ]
+                      },
+                      {
+                          "ArDO": [
+                              {
+                                  "ApduArDO": {
+                                      "generic_access_rule": "always"
+                                  }
+                              },
+                              {
+                                  "PermArDO": {
+                                      "permissions": "0000000000000001"
+                                  }
+                              }
+                          ]
+                      }
+                  ]
+              }
+          ]
+      }
+  ]
+
+aram_get_config
+~~~~~~~~~~~~~~~
+Perform Config handshake with ARA-M applet: Tell it our version and retrieve its version.
+
+NOTE: Not supported in all ARA-M implementations.
+
+
+aram_store_ref_ar_do
+~~~~~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.ara_m
+   :func: ADF_ARAM.AddlShellCommands.store_ref_ar_do_parse
+
+For example, to store an Android UICC carrier privilege rule for the SHA1 hash of the certificate used to sign the CoIMS android app of Supreeth Herle (https://github.com/herlesupreeth/CoIMS_Wiki) you can use the following command:
+::
+
+  pySIM-shell (MF/ADF.ARA-M)> aram_store_ref_ar_do --aid FFFFFFFFFFFF --device-app-id E46872F28B350B7E1F140DE535C2A8D5804F0BE3 --android-permissions 0000000000000001 --apdu-always
+
+
+aram_delete_all
+~~~~~~~~~~~~~~~
+This command will request deletion of all access rules stored within the
+ARA-M applet.  Use it with caution, there is no undo.  Any rules later
+intended must be manually inserted again using `aram_store_ref_ar_do`
+
+
+
+cmd2 settable parameters
+------------------------
+
+``cmd2`` has the concept of *settable parameters* which act a bit like environment variables in an OS-level
+shell: They can be read and set, and they will influence the behavior somehow.
+
+conserve_write
+~~~~~~~~~~~~~~
+
+If enabled, pySim will (when asked to write to a card) always first read the respective file/record and
+verify if the to-be-written value differs from the current on-card value.  If not, the write will be skipped.
+Writes will only be performed if the new value is different from the current on-card value.
+
+If disabled, pySim will always write irrespective of the current/new value.
+
+json_pretty_print
+~~~~~~~~~~~~~~~~~
+
+This parameter determines if generated JSON output should (by default) be pretty-printed (multi-line
+output with indent level of 4 spaces) or not.
+
+The default value of this parameter is 'true'.
+
+debug
+~~~~~
+
+If enabled, full python back-traces will be displayed in case of exceptions
+
+apdu_trace
+~~~~~~~~~~
+
+Boolean variable that determines if a hex-dump of the command + response APDU shall be printed.
+
+numeric_path
+~~~~~~~~~~~~
+
+Boolean variable that determines if path (e.g. in prompt) is displayed with numeric FIDs or string names.
+
+::
+
+  pySIM-shell (MF/EF.ICCID)> set numeric_path True
+  numeric_path - was: False
+  now: True
+  pySIM-shell (3f00/2fe2)> set numeric_path False
+  numeric_path - was: True
+  now: False
+  pySIM-shell (MF/EF.ICCID)> help set

ota_test.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+
+from pySim.ota import *
+from pySim.sms import SMS_SUBMIT, SMS_DELIVER, AddressField
+from pySim.utils import h2b, h2b
+
+# KIC1 + KID1 of 8988211000000515398
+#KIC1 = h2b('C039ED58F7B81446105E79EBFD373038')
+#KID1 = h2b('1799B93FE53F430BD7FD4810C77E1FDF')
+#KIC3 = h2b('167F2576D64C8D41862954875C8D7979')
+#KID3 = h2b('ECAE122B0E6AE4186D6487D50FDC0922')
+
+# KIC1 + KID1 of 8988211000000467285
+KIC1 = h2b('D0FDA31990D8D64178601317191669B4')
+KID1 = h2b('D24EB461799C5E035C77451FD9404463')
+KIC3 = h2b('C21DD66ACAC13CB3BC8B331B24AFB57B')
+KID3 = h2b('12110C78E678C25408233076AA033615')
+
+od = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3, kic=KIC3,
+               algo_auth='triple_des_cbc2', kid_idx=3, kid=KID3)
+print(od.crypt)
+print(od.auth)
+
+dialect = OtaDialectSms()
+
+# RAM: B00000
+# SIM RFM: B00010
+# USIM RFM: B00011
+tar = h2b('B00011')
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':True, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+outp = dialect.encode_cmd(od, tar, spi, apdu=b'\x00\xa4\x00\x04\x02\x3f\x00')
+print("result: %s" % b2h(outp))
+
+with_udh = b'\x02\x70\x00' + outp
+print("with_udh: %s" % b2h(with_udh))
+
+
+da = AddressField('12345678', 'unknown', 'isdn_e164')
+#tpdu = SMS_SUBMIT(tp_udhi=True, tp_mr=0x23, tp_da=da, tp_pid=0x7F, tp_dcs=0xF6, tp_udl=3, tp_ud=with_udh)
+tpdu = SMS_DELIVER(tp_udhi=True, tp_oa=da, tp_pid=0x7F, tp_dcs=0xF6, tp_scts=h2b('22705200000000'), tp_udl=3, tp_ud=with_udh)
+print(tpdu)
+print("tpdu: %s" % b2h(tpdu.toBytes()))
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':True, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+dialect.decode_resp(od, spi, '027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c')
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':False, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+dialect.decode_resp(od, spi, '027100001612b000110000000000000055f47118381175fb01612f')
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':False, 'por_rc_cc_ds': 'no_rc_cc_ds', 'por': 'por_required'}
+dialect.decode_resp(od, spi, '027100000e0ab000110000000000000001612f')
+

pySim-prog.py

@@ -0,0 +1,799 @@
+#!/usr/bin/env python3
+
+#
+# Utility to deal with sim cards and program the 'magic' ones easily
+#
+#
+# Part of the sim link code of inspired by pySimReader-Serial-src-v2
+#
+#
+# Copyright (C) 2009  Sylvain Munaut <tnt@246tNt.com>
+# Copyright (C) 2010  Harald Welte <laforge@gnumonks.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import hashlib
+from optparse import OptionParser
+import os
+import random
+import re
+import sys
+import traceback
+import json
+
+from pySim.commands import SimCardCommands
+from pySim.transport import init_reader
+from pySim.cards import _cards_classes, card_detect
+from pySim.utils import h2b, swap_nibbles, rpad, derive_milenage_opc, calculate_luhn, dec_iccid
+from pySim.ts_51_011 import EF, EF_AD
+from pySim.card_handler import *
+from pySim.utils import *
+
+
+def parse_options():
+
+    parser = OptionParser(usage="usage: %prog [options]")
+
+    parser.add_option("-d", "--device", dest="device", metavar="DEV",
+                      help="Serial Device for SIM access [default: %default]",
+                      default="/dev/ttyUSB0",
+                      )
+    parser.add_option("-b", "--baud", dest="baudrate", type="int", metavar="BAUD",
+                      help="Baudrate used for SIM access [default: %default]",
+                      default=9600,
+                      )
+    parser.add_option("-p", "--pcsc-device", dest="pcsc_dev", type='int', metavar="PCSC",
+                      help="Which PC/SC reader number for SIM access",
+                      default=None,
+                      )
+    parser.add_option("--modem-device", dest="modem_dev", metavar="DEV",
+                      help="Serial port of modem for Generic SIM Access (3GPP TS 27.007)",
+                      default=None,
+                      )
+    parser.add_option("--modem-baud", dest="modem_baud", type="int", metavar="BAUD",
+                      help="Baudrate used for modem's port [default: %default]",
+                      default=115200,
+                      )
+    parser.add_option("--osmocon", dest="osmocon_sock", metavar="PATH",
+                      help="Socket path for Calypso (e.g. Motorola C1XX) based reader (via OsmocomBB)",
+                      default=None,
+                      )
+    parser.add_option("-t", "--type", dest="type",
+                      help="Card type (user -t list to view) [default: %default]",
+                      default="auto",
+                      )
+    parser.add_option("-T", "--probe", dest="probe",
+                      help="Determine card type",
+                      default=False, action="store_true"
+                      )
+    parser.add_option("-a", "--pin-adm", dest="pin_adm",
+                      help="ADM PIN used for provisioning (overwrites default)",
+                      )
+    parser.add_option("-A", "--pin-adm-hex", dest="pin_adm_hex",
+                      help="ADM PIN used for provisioning, as hex string (16 characters long",
+                      )
+    parser.add_option("-e", "--erase", dest="erase", action='store_true',
+                      help="Erase beforehand [default: %default]",
+                      default=False,
+                      )
+
+    parser.add_option("-S", "--source", dest="source",
+                      help="Data Source[default: %default]",
+                      default="cmdline",
+                      )
+
+    # if mode is "cmdline"
+    parser.add_option("-n", "--name", dest="name",
+                      help="Operator name [default: %default]",
+                      default="Magic",
+                      )
+    parser.add_option("-c", "--country", dest="country", type="int", metavar="CC",
+                      help="Country code [default: %default]",
+                      default=1,
+                      )
+    parser.add_option("-x", "--mcc", dest="mcc", type="string",
+                      help="Mobile Country Code [default: %default]",
+                      default="901",
+                      )
+    parser.add_option("-y", "--mnc", dest="mnc", type="string",
+                      help="Mobile Network Code [default: %default]",
+                      default="55",
+                      )
+    parser.add_option("--mnclen", dest="mnclen", type="choice",
+                      help="Length of Mobile Network Code [default: %default]",
+                      default=2,
+                      choices=[2, 3],
+                      )
+    parser.add_option("-m", "--smsc", dest="smsc",
+                      help="SMSC number (Start with + for international no.) [default: '00 + country code + 5555']",
+                      )
+    parser.add_option("-M", "--smsp", dest="smsp",
+                      help="Raw SMSP content in hex [default: auto from SMSC]",
+                      )
+
+    parser.add_option("-s", "--iccid", dest="iccid", metavar="ID",
+                      help="Integrated Circuit Card ID",
+                      )
+    parser.add_option("-i", "--imsi", dest="imsi",
+                      help="International Mobile Subscriber Identity",
+                      )
+    parser.add_option("--msisdn", dest="msisdn",
+                      help="Mobile Subscriber Integrated Services Digital Number",
+                      )
+    parser.add_option("-k", "--ki", dest="ki",
+                      help="Ki (default is to randomize)",
+                      )
+    parser.add_option("-o", "--opc", dest="opc",
+                      help="OPC (default is to randomize)",
+                      )
+    parser.add_option("--op", dest="op",
+                      help="Set OP to derive OPC from OP and KI",
+                      )
+    parser.add_option("--acc", dest="acc",
+                      help="Set ACC bits (Access Control Code). not all card types are supported",
+                      )
+    parser.add_option("--opmode", dest="opmode", type="choice",
+                      help="Set UE Operation Mode in EF.AD (Administrative Data)",
+                      default=None,
+                      choices=['{:02X}'.format(int(m)) for m in EF_AD.OP_MODE],
+                      )
+    parser.add_option("--epdgid", dest="epdgid",
+                      help="Set Home Evolved Packet Data Gateway (ePDG) Identifier. (Only FQDN format supported)",
+                      )
+    parser.add_option("--epdgSelection", dest="epdgSelection",
+                      help="Set PLMN for ePDG Selection Information. (Only Operator Identifier FQDN format supported)",
+                      )
+    parser.add_option("--pcscf", dest="pcscf",
+                      help="Set Proxy Call Session Control Function (P-CSCF) Address. (Only FQDN format supported)",
+                      )
+    parser.add_option("--ims-hdomain", dest="ims_hdomain",
+                      help="Set IMS Home Network Domain Name in FQDN format",
+                      )
+    parser.add_option("--impi", dest="impi",
+                      help="Set IMS private user identity",
+                      )
+    parser.add_option("--impu", dest="impu",
+                      help="Set IMS public user identity",
+                      )
+    parser.add_option("--read-imsi", dest="read_imsi", action="store_true",
+                      help="Read the IMSI from the CARD", default=False
+                      )
+    parser.add_option("--read-iccid", dest="read_iccid", action="store_true",
+                      help="Read the ICCID from the CARD", default=False
+                      )
+    parser.add_option("-z", "--secret", dest="secret", metavar="STR",
+                      help="Secret used for ICCID/IMSI autogen",
+                      )
+    parser.add_option("-j", "--num", dest="num", type=int,
+                      help="Card # used for ICCID/IMSI autogen",
+                      )
+    parser.add_option("--batch", dest="batch_mode",
+                      help="Enable batch mode [default: %default]",
+                      default=False, action='store_true',
+                      )
+    parser.add_option("--batch-state", dest="batch_state", metavar="FILE",
+                      help="Optional batch state file",
+                      )
+
+    # if mode is "csv"
+    parser.add_option("--read-csv", dest="read_csv", metavar="FILE",
+                      help="Read parameters from CSV file rather than command line")
+
+    parser.add_option("--write-csv", dest="write_csv", metavar="FILE",
+                      help="Append generated parameters in CSV file",
+                      )
+    parser.add_option("--write-hlr", dest="write_hlr", metavar="FILE",
+                      help="Append generated parameters to OpenBSC HLR sqlite3",
+                      )
+    parser.add_option("--dry-run", dest="dry_run",
+                      help="Perform a 'dry run', don't actually program the card",
+                      default=False, action="store_true")
+    parser.add_option("--card_handler", dest="card_handler_config", metavar="FILE",
+                      help="Use automatic card handling machine")
+
+    (options, args) = parser.parse_args()
+
+    if options.type == 'list':
+        for kls in _cards_classes:
+            print(kls.name)
+        sys.exit(0)
+
+    if options.probe:
+        return options
+
+    if options.source == 'csv':
+        if (options.imsi is None) and (options.batch_mode is False) and (options.read_imsi is False) and (options.read_iccid is False):
+            parser.error(
+                "CSV mode needs either an IMSI, --read-imsi, --read-iccid or batch mode")
+        if options.read_csv is None:
+            parser.error("CSV mode requires a CSV input file")
+    elif options.source == 'cmdline':
+        if ((options.imsi is None) or (options.iccid is None)) and (options.num is None):
+            parser.error(
+                "If either IMSI or ICCID isn't specified, num is required")
+    else:
+        parser.error("Only `cmdline' and `csv' sources supported")
+
+    if (options.read_csv is not None) and (options.source != 'csv'):
+        parser.error("You cannot specify a CSV input file in source != csv")
+
+    if (options.batch_mode) and (options.num is None):
+        options.num = 0
+
+    if (options.batch_mode):
+        if (options.imsi is not None) or (options.iccid is not None):
+            parser.error(
+                "Can't give ICCID/IMSI for batch mode, need to use automatic parameters ! see --num and --secret for more information")
+
+    if args:
+        parser.error("Extraneous arguments")
+
+    return options
+
+
+def _digits(secret, usage, len, num):
+    seed = secret + usage + '%d' % num
+    s = hashlib.sha1(seed.encode())
+    d = ''.join(['%02d' % x for x in s.digest()])
+    return d[0:len]
+
+
+def _mcc_mnc_digits(mcc, mnc):
+    return '%s%s' % (mcc, mnc)
+
+
+def _cc_digits(cc):
+    return ('%03d' if cc > 100 else '%02d') % cc
+
+
+def _isnum(s, l=-1):
+    return s.isdigit() and ((l == -1) or (len(s) == l))
+
+
+def _ishex(s, l=-1):
+    hc = '0123456789abcdef'
+    return all([x in hc for x in s.lower()]) and ((l == -1) or (len(s) == l))
+
+
+def _dbi_binary_quote(s):
+    # Count usage of each char
+    cnt = {}
+    for c in s:
+        cnt[c] = cnt.get(c, 0) + 1
+
+    # Find best offset
+    e = 0
+    m = len(s)
+    for i in range(1, 256):
+        if i == 39:
+            continue
+        sum_ = cnt.get(i, 0) + cnt.get((i+1) & 0xff, 0) + \
+            cnt.get((i+39) & 0xff, 0)
+        if sum_ < m:
+            m = sum_
+            e = i
+            if m == 0:  # No overhead ? use this !
+                break
+
+    # Generate output
+    out = []
+    out.append(chr(e))  # Offset
+    for c in s:
+        x = (256 + ord(c) - e) % 256
+        if x in (0, 1, 39):
+            out.append('\x01')
+            out.append(chr(x+1))
+        else:
+            out.append(chr(x))
+
+    return ''.join(out)
+
+
+def gen_parameters(opts):
+    """Generates Name, ICCID, MCC, MNC, IMSI, SMSP, Ki, PIN-ADM from the
+    options given by the user"""
+
+    # MCC/MNC
+    mcc = opts.mcc
+    mnc = opts.mnc
+
+    if not mcc.isdigit() or not mnc.isdigit():
+        raise ValueError('mcc & mnc must only contain decimal digits')
+    if len(mcc) < 1 or len(mcc) > 3:
+        raise ValueError('mcc must be between 1 .. 3 digits')
+    if len(mnc) < 1 or len(mnc) > 3:
+        raise ValueError('mnc must be between 1 .. 3 digits')
+
+    # MCC always has 3 digits
+    mcc = lpad(mcc, 3, "0")
+    # MNC must be at least 2 digits
+    mnc = lpad(mnc, 2, "0")
+
+    # Digitize country code (2 or 3 digits)
+    cc_digits = _cc_digits(opts.country)
+
+    # Digitize MCC/MNC (5 or 6 digits)
+    plmn_digits = _mcc_mnc_digits(mcc, mnc)
+
+    if opts.name is not None:
+        if len(opts.name) > 16:
+            raise ValueError('Service Provider Name must max 16 characters!')
+
+    if opts.msisdn is not None:
+        msisdn = opts.msisdn
+        if msisdn[0] == '+':
+            msisdn = msisdn[1:]
+        if not msisdn.isdigit():
+            raise ValueError('MSISDN must be digits only! '
+                             'Start with \'+\' for international numbers.')
+        if len(msisdn) > 10 * 2:
+            # TODO: Support MSISDN of length > 20 (10 Bytes)
+            raise ValueError(
+                'MSISDNs longer than 20 digits are not (yet) supported.')
+
+    # ICCID (19 digits, E.118), though some phase1 vendors use 20 :(
+    if opts.iccid is not None:
+        iccid = opts.iccid
+        if not _isnum(iccid, 19) and not _isnum(iccid, 20):
+            raise ValueError('ICCID must be 19 or 20 digits !')
+
+    else:
+        if opts.num is None:
+            raise ValueError('Neither ICCID nor card number specified !')
+
+        iccid = (
+            '89' +			# Common prefix (telecom)
+            cc_digits +		# Country Code on 2/3 digits
+            plmn_digits 		# MCC/MNC on 5/6 digits
+        )
+
+        ml = 18 - len(iccid)
+
+        if opts.secret is None:
+            # The raw number
+            iccid += ('%%0%dd' % ml) % opts.num
+        else:
+            # Randomized digits
+            iccid += _digits(opts.secret, 'ccid', ml, opts.num)
+
+        # Add checksum digit
+        iccid += ('%1d' % calculate_luhn(iccid))
+
+    # IMSI (15 digits usually)
+    if opts.imsi is not None:
+        imsi = opts.imsi
+        if not _isnum(imsi):
+            raise ValueError('IMSI must be digits only !')
+
+    else:
+        if opts.num is None:
+            raise ValueError('Neither IMSI nor card number specified !')
+
+        ml = 15 - len(plmn_digits)
+
+        if opts.secret is None:
+            # The raw number
+            msin = ('%%0%dd' % ml) % opts.num
+        else:
+            # Randomized digits
+            msin = _digits(opts.secret, 'imsi', ml, opts.num)
+
+        imsi = (
+            plmn_digits +  # MCC/MNC on 5/6 digits
+            msin			# MSIN
+        )
+
+    # SMSP
+    if opts.smsp is not None:
+        smsp = opts.smsp
+        if not _ishex(smsp):
+            raise ValueError('SMSP must be hex digits only !')
+        if len(smsp) < 28*2:
+            raise ValueError('SMSP must be at least 28 bytes')
+
+    else:
+        ton = "81"
+        if opts.smsc is not None:
+            smsc = opts.smsc
+            if smsc[0] == '+':
+                ton = "91"
+                smsc = smsc[1:]
+            if not _isnum(smsc):
+                raise ValueError('SMSC must be digits only!\n \
+					Start with \'+\' for international numbers')
+        else:
+            smsc = '00%d' % opts.country + '5555'  # Hack ...
+
+        smsc = '%02d' % ((len(smsc) + 3)//2,) + ton + \
+            swap_nibbles(rpad(smsc, 20))
+
+        smsp = (
+            'e1' +			# Parameters indicator
+            'ff' * 12 +		# TP-Destination address
+            smsc +			# TP-Service Centre Address
+            '00' +			# TP-Protocol identifier
+            '00' +			# TP-Data coding scheme
+            '00'			# TP-Validity period
+        )
+
+    # ACC
+    if opts.acc is not None:
+        acc = opts.acc
+        if not _ishex(acc):
+            raise ValueError('ACC must be hex digits only !')
+        if len(acc) != 2*2:
+            raise ValueError('ACC must be exactly 2 bytes')
+
+    else:
+        acc = None
+
+    # Ki (random)
+    if opts.ki is not None:
+        ki = opts.ki
+        if not re.match('^[0-9a-fA-F]{32}$', ki):
+            raise ValueError('Ki needs to be 128 bits, in hex format')
+    else:
+        ki = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
+
+    # OPC (random)
+    if opts.opc is not None:
+        opc = opts.opc
+        if not re.match('^[0-9a-fA-F]{32}$', opc):
+            raise ValueError('OPC needs to be 128 bits, in hex format')
+
+    elif opts.op is not None:
+        opc = derive_milenage_opc(ki, opts.op)
+    else:
+        opc = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
+
+    pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
+
+    # ePDG Selection Information
+    if opts.epdgSelection:
+        if len(opts.epdgSelection) < 5 or len(opts.epdgSelection) > 6:
+            raise ValueError('ePDG Selection Information is not valid')
+        epdg_mcc = opts.epdgSelection[:3]
+        epdg_mnc = opts.epdgSelection[3:]
+        if not epdg_mcc.isdigit() or not epdg_mnc.isdigit():
+            raise ValueError(
+                'PLMN for ePDG Selection must only contain decimal digits')
+
+    # Return that
+    return {
+        'name': opts.name,
+        'iccid': iccid,
+        'mcc': mcc,
+        'mnc': mnc,
+        'imsi': imsi,
+        'smsp': smsp,
+        'ki': ki,
+        'opc': opc,
+        'acc': acc,
+        'pin_adm': pin_adm,
+        'msisdn': opts.msisdn,
+        'epdgid': opts.epdgid,
+        'epdgSelection': opts.epdgSelection,
+        'pcscf': opts.pcscf,
+        'ims_hdomain': opts.ims_hdomain,
+        'impi': opts.impi,
+        'impu': opts.impu,
+        'opmode': opts.opmode,
+    }
+
+
+def print_parameters(params):
+
+    s = ["Generated card parameters :"]
+    if 'name' in params:
+        s.append(" > Name     : %(name)s")
+    if 'smsp' in params:
+        s.append(" > SMSP     : %(smsp)s")
+    s.append(" > ICCID    : %(iccid)s")
+    s.append(" > MCC/MNC  : %(mcc)s/%(mnc)s")
+    s.append(" > IMSI     : %(imsi)s")
+    s.append(" > Ki       : %(ki)s")
+    s.append(" > OPC      : %(opc)s")
+    if 'acc' in params:
+        s.append(" > ACC      : %(acc)s")
+    s.append(" > ADM1(hex): %(pin_adm)s")
+    if 'opmode' in params:
+        s.append(" > OPMODE   : %(opmode)s")
+    print("\n".join(s) % params)
+
+
+def write_params_csv(opts, params):
+    # csv
+    if opts.write_csv:
+        import csv
+        row = ['name', 'iccid', 'mcc', 'mnc', 'imsi', 'smsp', 'ki', 'opc']
+        f = open(opts.write_csv, 'a')
+        cw = csv.writer(f)
+        cw.writerow([params[x] for x in row])
+        f.close()
+
+
+def _read_params_csv(opts, iccid=None, imsi=None):
+    import csv
+    f = open(opts.read_csv, 'r')
+    cr = csv.DictReader(f)
+
+    # Lower-case fieldnames
+    cr.fieldnames = [field.lower() for field in cr.fieldnames]
+
+    i = 0
+    if not 'iccid' in cr.fieldnames:
+        raise Exception("CSV file in wrong format!")
+    for row in cr:
+        if opts.num is not None and opts.read_iccid is False and opts.read_imsi is False:
+            if opts.num == i:
+                f.close()
+                return row
+            i += 1
+        if row['iccid'] == iccid:
+            f.close()
+            return row
+
+        if row['imsi'] == imsi:
+            f.close()
+            return row
+
+    f.close()
+    return None
+
+
+def read_params_csv(opts, imsi=None, iccid=None):
+    row = _read_params_csv(opts, iccid=iccid, imsi=imsi)
+    if row is not None:
+        row['mcc'] = row.get('mcc', mcc_from_imsi(row.get('imsi')))
+        row['mnc'] = row.get('mnc', mnc_from_imsi(row.get('imsi')))
+
+        pin_adm = None
+        # We need to escape the pin_adm we get from the csv
+        if 'pin_adm' in row:
+            pin_adm = ''.join(['%02x' % (ord(x)) for x in row['pin_adm']])
+        # Stay compatible to the odoo csv format
+        elif 'adm1' in row:
+            pin_adm = ''.join(['%02x' % (ord(x)) for x in row['adm1']])
+        if pin_adm:
+            row['pin_adm'] = rpad(pin_adm, 16)
+
+        # If the CSV-File defines a pin_adm_hex field use this field to
+        # generate pin_adm from that.
+        pin_adm_hex = row.get('pin_adm_hex')
+        if pin_adm_hex:
+            if len(pin_adm_hex) == 16:
+                row['pin_adm'] = pin_adm_hex
+                # Ensure that it's hex-encoded
+                try:
+                    try_encode = h2b(pin_adm)
+                except ValueError:
+                    raise ValueError(
+                        "pin_adm_hex needs to be hex encoded using this option")
+            else:
+                raise ValueError(
+                    "pin_adm_hex needs to be exactly 16 digits (hex encoded)")
+
+    return row
+
+
+def write_params_hlr(opts, params):
+    # SQLite3 OpenBSC HLR
+    if opts.write_hlr:
+        import sqlite3
+        conn = sqlite3.connect(opts.write_hlr)
+
+        c = conn.execute(
+            'INSERT INTO Subscriber ' +
+            '(imsi, name, extension, authorized, created, updated) ' +
+            'VALUES ' +
+            '(?,?,?,1,datetime(\'now\'),datetime(\'now\'));',
+            [
+                params['imsi'],
+                params['name'],
+                '9' + params['iccid'][-5:-1]
+            ],
+        )
+        sub_id = c.lastrowid
+        c.close()
+
+        c = conn.execute(
+            'INSERT INTO AuthKeys ' +
+            '(subscriber_id, algorithm_id, a3a8_ki)' +
+            'VALUES ' +
+            '(?,?,?)',
+            [sub_id, 2, sqlite3.Binary(
+                _dbi_binary_quote(h2b(params['ki'])))],
+        )
+
+        conn.commit()
+        conn.close()
+
+
+def write_parameters(opts, params):
+    write_params_csv(opts, params)
+    write_params_hlr(opts, params)
+
+
+BATCH_STATE = ['name', 'country', 'mcc', 'mnc', 'smsp', 'secret', 'num']
+BATCH_INCOMPATIBLE = ['iccid', 'imsi', 'ki']
+
+
+def init_batch(opts):
+    # Need to do something ?
+    if not opts.batch_mode:
+        return
+
+    for k in BATCH_INCOMPATIBLE:
+        if getattr(opts, k):
+            print("Incompatible option with batch_state: %s" % (k,))
+            sys.exit(-1)
+
+    # Don't load state if there is none ...
+    if not opts.batch_state:
+        return
+
+    if not os.path.isfile(opts.batch_state):
+        print("No state file yet")
+        return
+
+    # Get stored data
+    fh = open(opts.batch_state)
+    d = json.loads(fh.read())
+    fh.close()
+
+    for k, v in d.iteritems():
+        setattr(opts, k, v)
+
+
+def save_batch(opts):
+    # Need to do something ?
+    if not opts.batch_mode or not opts.batch_state:
+        return
+
+    d = json.dumps(dict([(k, getattr(opts, k)) for k in BATCH_STATE]))
+    fh = open(opts.batch_state, 'w')
+    fh.write(d)
+    fh.close()
+
+
+def process_card(opts, first, ch):
+
+    if opts.dry_run is False:
+        # Connect transport
+        ch.get(first)
+
+    if opts.dry_run is False:
+        # Get card
+        card = card_detect(opts.type, scc)
+        if card is None:
+            print("No card detected!")
+            return -1
+
+        # Probe only
+        if opts.probe:
+            return 0
+
+        # Erase if requested
+        if opts.erase:
+            print("Formatting ...")
+            card.erase()
+            card.reset()
+
+    # Generate parameters
+    if opts.source == 'cmdline':
+        cp = gen_parameters(opts)
+    elif opts.source == 'csv':
+        imsi = None
+        iccid = None
+        if opts.read_iccid:
+            if opts.dry_run:
+                # Connect transport
+                ch.get(False)
+            (res, _) = scc.read_binary(['3f00', '2fe2'], length=10)
+            iccid = dec_iccid(res)
+        elif opts.read_imsi:
+            if opts.dry_run:
+                # Connect transport
+                ch.get(False)
+            (res, _) = scc.read_binary(EF['IMSI'])
+            imsi = swap_nibbles(res)[3:]
+        else:
+            imsi = opts.imsi
+        cp = read_params_csv(opts, imsi=imsi, iccid=iccid)
+    if cp is None:
+        print("Error reading parameters from CSV file!\n")
+        return 2
+    print_parameters(cp)
+
+    if opts.dry_run is False:
+        # Program the card
+        print("Programming ...")
+        card.program(cp)
+    else:
+        print("Dry Run: NOT PROGRAMMING!")
+
+    # Write parameters permanently
+    write_parameters(opts, cp)
+
+    # Batch mode state update and save
+    if opts.num is not None:
+        opts.num += 1
+    save_batch(opts)
+
+    ch.done()
+    return 0
+
+
+if __name__ == '__main__':
+
+    # Parse options
+    opts = parse_options()
+
+    # Init card reader driver
+    sl = init_reader(opts)
+    if sl is None:
+        exit(1)
+
+    # Create command layer
+    scc = SimCardCommands(transport=sl)
+
+    # If we use a CSV file as data input, check if the CSV file exists.
+    if opts.source == 'csv':
+        print("Using CSV file as data input: " + str(opts.read_csv))
+        if not os.path.isfile(opts.read_csv):
+            print("CSV file not found!")
+            sys.exit(1)
+
+    # Batch mode init
+    init_batch(opts)
+
+    if opts.card_handler_config:
+        ch = CardHandlerAuto(sl, opts.card_handler_config)
+    else:
+        ch = CardHandler(sl)
+
+    # Iterate
+    first = True
+    card = None
+
+    while 1:
+        try:
+            rc = process_card(opts, first, ch)
+        except (KeyboardInterrupt):
+            print("")
+            print("Terminated by user!")
+            sys.exit(0)
+        except (SystemExit):
+            raise
+        except:
+            print("")
+            print("Card programming failed with an exception:")
+            print("---------------------8<---------------------")
+            traceback.print_exc()
+            print("---------------------8<---------------------")
+            print("")
+            rc = -1
+
+        # Something did not work as well as expected, however, lets
+        # make sure the card is pulled from the reader.
+        if rc != 0:
+            ch.error()
+
+        # If we are not in batch mode we are done in any case, so lets
+        # exit here.
+        if not opts.batch_mode:
+            sys.exit(rc)
+
+        first = False

pySim-read.py

@@ -0,0 +1,362 @@
+#!/usr/bin/env python3
+
+#
+# Utility to display some informations about a SIM card
+#
+#
+# Copyright (C) 2009  Sylvain Munaut <tnt@246tNt.com>
+# Copyright (C) 2010  Harald Welte <laforge@gnumonks.org>
+# Copyright (C) 2013  Alexander Chemeris <alexander.chemeris@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import hashlib
+import argparse
+import os
+import random
+import re
+import sys
+from pySim.ts_51_011 import EF, DF, EF_SST_map, EF_AD
+from pySim.ts_31_102 import EF_UST_map, EF_USIM_ADF_map
+from pySim.ts_31_103 import EF_IST_map, EF_ISIM_ADF_map
+
+from pySim.commands import SimCardCommands
+from pySim.transport import init_reader, argparse_add_reader_args
+from pySim.exceptions import SwMatchError
+from pySim.cards import card_detect, SimCard, UsimCard, IsimCard
+from pySim.utils import h2b, swap_nibbles, rpad, dec_imsi, dec_iccid, dec_msisdn
+from pySim.utils import format_xplmn_w_act, dec_st
+from pySim.utils import h2s, format_ePDGSelection
+
+option_parser = argparse.ArgumentParser(prog='pySim-read',
+                                        description='Legacy tool for reading some parts of a SIM card',
+                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+argparse_add_reader_args(option_parser)
+
+
+def select_app(adf: str, card: SimCard):
+    """Select application by its AID"""
+    sw = 0
+    try:
+        if card._scc.cla_byte == "00":
+            data, sw = card.select_adf_by_aid(adf)
+    except SwMatchError as e:
+        if e.sw_actual == "6a82":
+            # If we can't select the file because it does not exist, we just remain silent since it means
+            # that this card just does not have an USIM application installed, which is not an error.
+            pass
+        else:
+            print("ADF." + adf + ": Can't select application -- " + str(e))
+    except Exception as e:
+        print("ADF." + adf + ": Can't select application -- " + str(e))
+
+    return sw
+
+
+if __name__ == '__main__':
+
+    # Parse options
+    opts = option_parser.parse_args()
+
+    # Init card reader driver
+    sl = init_reader(opts)
+    if sl is None:
+        exit(1)
+
+    # Create command layer
+    scc = SimCardCommands(transport=sl)
+
+    # Wait for SIM card
+    sl.wait_for_card()
+
+    # Assuming UICC SIM
+    scc.cla_byte = "00"
+    scc.sel_ctrl = "0004"
+
+    # Testing for Classic SIM or UICC
+    (res, sw) = sl.send_apdu(scc.cla_byte + "a4" + scc.sel_ctrl + "02" + "3f00")
+    if sw == '6e00':
+        # Just a Classic SIM
+        scc.cla_byte = "a0"
+        scc.sel_ctrl = "0000"
+
+    # Read the card
+    print("Reading ...")
+
+    # Initialize Card object by auto detecting the card
+    card = card_detect("auto", scc) or SimCard(scc)
+
+    # Read all AIDs on the UICC
+    card.read_aids()
+
+    # EF.ICCID
+    (res, sw) = card.read_iccid()
+    if sw == '9000':
+        print("ICCID: %s" % (res,))
+    else:
+        print("ICCID: Can't read, response code = %s" % (sw,))
+
+    # EF.IMSI
+    (res, sw) = card.read_imsi()
+    if sw == '9000':
+        print("IMSI: %s" % (res,))
+    else:
+        print("IMSI: Can't read, response code = %s" % (sw,))
+
+    # EF.GID1
+    try:
+        (res, sw) = card.read_gid1()
+        if sw == '9000':
+            print("GID1: %s" % (res,))
+        else:
+            print("GID1: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("GID1: Can't read file -- %s" % (str(e),))
+
+    # EF.GID2
+    try:
+        (res, sw) = card.read_binary('GID2')
+        if sw == '9000':
+            print("GID2: %s" % (res,))
+        else:
+            print("GID2: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("GID2: Can't read file -- %s" % (str(e),))
+
+    # EF.SMSP
+    (res, sw) = card.read_record('SMSP', 1)
+    if sw == '9000':
+        print("SMSP: %s" % (res,))
+    else:
+        print("SMSP: Can't read, response code = %s" % (sw,))
+
+    # EF.SPN
+    try:
+        (res, sw) = card.read_spn()
+        if sw == '9000':
+            print("SPN: %s" % (res[0] or "Not available"))
+            print("Show in HPLMN: %s" % (res[1],))
+            print("Hide in OPLMN: %s" % (res[2],))
+        else:
+            print("SPN: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("SPN: Can't read file -- %s" % (str(e),))
+
+    # EF.PLMNsel
+    try:
+        (res, sw) = card.read_binary('PLMNsel')
+        if sw == '9000':
+            print("PLMNsel: %s" % (res))
+        else:
+            print("PLMNsel: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("PLMNsel: Can't read file -- " + str(e))
+
+    # EF.PLMNwAcT
+    try:
+        (res, sw) = card.read_plmn_act()
+        if sw == '9000':
+            print("PLMNwAcT:\n%s" % (res))
+        else:
+            print("PLMNwAcT: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("PLMNwAcT: Can't read file -- " + str(e))
+
+    # EF.OPLMNwAcT
+    try:
+        (res, sw) = card.read_oplmn_act()
+        if sw == '9000':
+            print("OPLMNwAcT:\n%s" % (res))
+        else:
+            print("OPLMNwAcT: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("OPLMNwAcT: Can't read file -- " + str(e))
+
+    # EF.HPLMNAcT
+    try:
+        (res, sw) = card.read_hplmn_act()
+        if sw == '9000':
+            print("HPLMNAcT:\n%s" % (res))
+        else:
+            print("HPLMNAcT: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("HPLMNAcT: Can't read file -- " + str(e))
+
+    # EF.ACC
+    (res, sw) = card.read_binary('ACC')
+    if sw == '9000':
+        print("ACC: %s" % (res,))
+    else:
+        print("ACC: Can't read, response code = %s" % (sw,))
+
+    # EF.MSISDN
+    try:
+        (res, sw) = card.read_msisdn()
+        if sw == '9000':
+            # (npi, ton, msisdn) = res
+            if res is not None:
+                print("MSISDN (NPI=%d ToN=%d): %s" % res)
+            else:
+                print("MSISDN: Not available")
+        else:
+            print("MSISDN: Can't read, response code = %s" % (sw,))
+    except Exception as e:
+        print("MSISDN: Can't read file -- " + str(e))
+
+    # EF.AD
+    (res, sw) = card.read_binary('AD')
+    if sw == '9000':
+        print("Administrative data: %s" % (res,))
+        ad = EF_AD()
+        decoded_data = ad.decode_hex(res)
+        print("\tMS operation mode: %s" % decoded_data['ms_operation_mode'])
+        if decoded_data['ofm']:
+            print("\tCiphering Indicator: enabled")
+        else:
+            print("\tCiphering Indicator: disabled")
+    else:
+        print("AD: Can't read, response code = %s" % (sw,))
+
+    # EF.SST
+    (res, sw) = card.read_binary('SST')
+    if sw == '9000':
+        print("SIM Service Table: %s" % res)
+        # Print those which are available
+        print("%s" % dec_st(res))
+    else:
+        print("SIM Service Table: Can't read, response code = %s" % (sw,))
+
+    # Check whether we have th AID of USIM, if so select it by its AID
+    # EF.UST - File Id in ADF USIM : 6f38
+    sw = select_app("USIM", card)
+    if sw == '9000':
+        # Select USIM profile
+        usim_card = UsimCard(scc)
+
+        # EF.EHPLMN
+        if usim_card.file_exists(EF_USIM_ADF_map['EHPLMN']):
+            (res, sw) = usim_card.read_ehplmn()
+            if sw == '9000':
+                print("EHPLMN:\n%s" % (res))
+            else:
+                print("EHPLMN: Can't read, response code = %s" % (sw,))
+
+        # EF.UST
+        try:
+            if usim_card.file_exists(EF_USIM_ADF_map['UST']):
+                # res[0] - EF content of UST
+                # res[1] - Human readable format of services marked available in UST
+                (res, sw) = usim_card.read_ust()
+                if sw == '9000':
+                    print("USIM Service Table: %s" % res[0])
+                    print("%s" % res[1])
+                else:
+                    print("USIM Service Table: Can't read, response code = %s" % (sw,))
+        except Exception as e:
+            print("USIM Service Table: Can't read file -- " + str(e))
+
+        # EF.ePDGId - Home ePDG Identifier
+        try:
+            if usim_card.file_exists(EF_USIM_ADF_map['ePDGId']):
+                (res, sw) = usim_card.read_epdgid()
+                if sw == '9000':
+                    print("ePDGId:\n%s" %
+                          (len(res) and res or '\tNot available\n',))
+                else:
+                    print("ePDGId: Can't read, response code = %s" % (sw,))
+        except Exception as e:
+            print("ePDGId: Can't read file -- " + str(e))
+
+        # EF.ePDGSelection - ePDG Selection Information
+        try:
+            if usim_card.file_exists(EF_USIM_ADF_map['ePDGSelection']):
+                (res, sw) = usim_card.read_ePDGSelection()
+                if sw == '9000':
+                    print("ePDGSelection:\n%s" % (res,))
+                else:
+                    print("ePDGSelection: Can't read, response code = %s" % (sw,))
+        except Exception as e:
+            print("ePDGSelection: Can't read file -- " + str(e))
+
+    # Select ISIM application by its AID
+    sw = select_app("ISIM", card)
+    if sw == '9000':
+        # Select USIM profile
+        isim_card = IsimCard(scc)
+
+        # EF.P-CSCF - P-CSCF Address
+        try:
+            if isim_card.file_exists(EF_ISIM_ADF_map['PCSCF']):
+                res = isim_card.read_pcscf()
+                print("P-CSCF:\n%s" %
+                      (len(res) and res or '\tNot available\n',))
+        except Exception as e:
+            print("P-CSCF: Can't read file -- " + str(e))
+
+        # EF.DOMAIN - Home Network Domain Name e.g. ims.mncXXX.mccXXX.3gppnetwork.org
+        try:
+            if isim_card.file_exists(EF_ISIM_ADF_map['DOMAIN']):
+                (res, sw) = isim_card.read_domain()
+                if sw == '9000':
+                    print("Home Network Domain Name: %s" %
+                          (len(res) and res or 'Not available',))
+                else:
+                    print(
+                        "Home Network Domain Name: Can't read, response code = %s" % (sw,))
+        except Exception as e:
+            print("Home Network Domain Name: Can't read file -- " + str(e))
+
+        # EF.IMPI - IMS private user identity
+        try:
+            if isim_card.file_exists(EF_ISIM_ADF_map['IMPI']):
+                (res, sw) = isim_card.read_impi()
+                if sw == '9000':
+                    print("IMS private user identity: %s" %
+                          (len(res) and res or 'Not available',))
+                else:
+                    print(
+                        "IMS private user identity: Can't read, response code = %s" % (sw,))
+        except Exception as e:
+            print("IMS private user identity: Can't read file -- " + str(e))
+
+        # EF.IMPU - IMS public user identity
+        try:
+            if isim_card.file_exists(EF_ISIM_ADF_map['IMPU']):
+                res = isim_card.read_impu()
+                print("IMS public user identity:\n%s" %
+                      (len(res) and res or '\tNot available\n',))
+        except Exception as e:
+            print("IMS public user identity: Can't read file -- " + str(e))
+
+        # EF.UICCIARI - UICC IARI
+        try:
+            if isim_card.file_exists(EF_ISIM_ADF_map['UICCIARI']):
+                res = isim_card.read_iari()
+                print("UICC IARI:\n%s" %
+                      (len(res) and res or '\tNot available\n',))
+        except Exception as e:
+            print("UICC IARI: Can't read file -- " + str(e))
+
+        # EF.IST
+        (res, sw) = card.read_binary('6f07')
+        if sw == '9000':
+            print("ISIM Service Table: %s" % res)
+            # Print those which are available
+            print("%s" % dec_st(res, table="isim"))
+        else:
+            print("ISIM Service Table: Can't read, response code = %s" % (sw,))
+
+    # Done for this card and maybe for everything ?
+    print("Done !\n")

pySim-shell.py

@@ -0,0 +1,997 @@
+#!/usr/bin/env python3
+
+# Interactive shell for working with SIM / UICC / USIM / ISIM cards
+#
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import List
+
+import json
+import traceback
+
+import cmd2
+from cmd2 import style, fg
+from cmd2 import CommandSet, with_default_category, with_argparser
+import argparse
+
+import os
+import sys
+from pathlib import Path
+from io import StringIO
+
+from pprint import pprint as pp
+
+from pySim.exceptions import *
+from pySim.commands import SimCardCommands
+from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args, ProactiveHandler
+from pySim.cards import card_detect, SimCard
+from pySim.utils import h2b, swap_nibbles, rpad, b2h, JsonEncoder, bertlv_parse_one, sw_match
+from pySim.utils import sanitize_pin_adm, tabulate_str_list, boxed_heading_str, Hexstr
+from pySim.card_handler import CardHandler, CardHandlerAuto
+
+from pySim.filesystem import RuntimeState, CardDF, CardADF, CardModel
+from pySim.profile import CardProfile
+from pySim.ts_102_221 import CardProfileUICC
+from pySim.ts_102_222 import Ts102222Commands
+from pySim.ts_31_102 import CardApplicationUSIM
+from pySim.ts_31_103 import CardApplicationISIM
+from pySim.ara_m import CardApplicationARAM
+from pySim.global_platform import CardApplicationISD
+from pySim.gsm_r import DF_EIRENE
+from pySim.cat import ProactiveCommand
+
+# we need to import this module so that the SysmocomSJA2 sub-class of
+# CardModel is created, which will add the ATR-based matching and
+# calling of SysmocomSJA2.add_files.  See  CardModel.apply_matching_models
+import pySim.sysmocom_sja2
+
+from pySim.card_key_provider import CardKeyProviderCsv, card_key_provider_register, card_key_provider_get_field
+
+
+def init_card(sl):
+    """
+    Detect card in reader and setup card profile and runtime state. This
+    function must be called at least once on startup. The card and runtime
+    state object (rs) is required for all pySim-shell commands.
+    """
+
+    # Wait up to three seconds for a card in reader and try to detect
+    # the card type.
+    print("Waiting for card...")
+    try:
+        sl.wait_for_card(3)
+    except NoCardError:
+        print("No card detected!")
+        return None, None
+    except:
+        print("Card not readable!")
+        return None, None
+
+    generic_card = False
+    card = card_detect("auto", scc)
+    if card is None:
+        print("Warning: Could not detect card type - assuming a generic card type...")
+        card = SimCard(scc)
+        generic_card = True
+
+    profile = CardProfile.pick(scc)
+    if profile is None:
+        print("Unsupported card type!")
+        return None, card
+
+    # ETSI TS 102 221, Table 9.3 specifies a default for the PIN key
+    # references, however card manufactures may still decide to pick an
+    # arbitrary key reference. In case we run on a generic card class that is
+    # detected as an UICC, we will pick the key reference that is officially
+    # specified.
+    if generic_card and isinstance(profile, CardProfileUICC):
+        card._adm_chv_num = 0x0A
+
+    print("Info: Card is of type: %s" % str(profile))
+
+    # FIXME: This shouln't be here, the profile should add the applications,
+    # however, we cannot simply put his into ts_102_221.py since we would
+    # have to e.g. import CardApplicationUSIM from ts_31_102.py, which already
+    # imports from ts_102_221.py. This means we will end up with a circular
+    # import, which needs to be resolved first.
+    if isinstance(profile, CardProfileUICC):
+        profile.add_application(CardApplicationUSIM())
+        profile.add_application(CardApplicationISIM())
+        profile.add_application(CardApplicationARAM())
+        profile.add_application(CardApplicationISD())
+
+    # Create runtime state with card profile
+    rs = RuntimeState(card, profile)
+
+    # FIXME: This is an GSM-R related file, it needs to be added throughout,
+    # the profile. At the moment we add it for all cards, this won't hurt,
+    # but regular SIM and UICC will not have it and fail to select it.
+    rs.mf.add_file(DF_EIRENE())
+
+    CardModel.apply_matching_models(scc, rs)
+
+    # inform the transport that we can do context-specific SW interpretation
+    sl.set_sw_interpreter(rs)
+
+    return rs, card
+
+
+class PysimApp(cmd2.Cmd):
+    CUSTOM_CATEGORY = 'pySim Commands'
+
+    def __init__(self, card, rs, sl, ch, script=None):
+        super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
+                         use_ipython=True, auto_load_commands=False, startup_script=script)
+        self.intro = style('Welcome to pySim-shell!', fg=fg.red)
+        self.default_category = 'pySim-shell built-in commands'
+        self.card = None
+        self.rs = None
+        self.lchan = None
+        self.py_locals = {'card': self.card, 'rs': self.rs, 'lchan': self.lchan}
+        self.sl = sl
+        self.ch = ch
+
+        self.numeric_path = False
+        self.add_settable(cmd2.Settable('numeric_path', bool, 'Print File IDs instead of names',
+                                        onchange_cb=self._onchange_numeric_path))
+        self.conserve_write = True
+        self.add_settable(cmd2.Settable('conserve_write', bool, 'Read and compare before write',
+                                        onchange_cb=self._onchange_conserve_write))
+        self.json_pretty_print = True
+        self.add_settable(cmd2.Settable('json_pretty_print',
+                          bool, 'Pretty-Print JSON output'))
+        self.apdu_trace = False
+        self.add_settable(cmd2.Settable('apdu_trace', bool, 'Trace and display APDUs exchanged with card',
+                                        onchange_cb=self._onchange_apdu_trace))
+
+        self.equip(card, rs)
+
+    def equip(self, card, rs):
+        """
+        Equip pySim-shell with the supplied card and runtime state, add (or remove) all required settables and
+        and commands to enable card operations.
+        """
+
+        rc = False
+
+        # Unequip everything from pySim-shell that would not work in unequipped state
+        if self.rs:
+            lchan = self.rs.lchan[0]
+            lchan.unregister_cmds(self)
+        for cmds in [Iso7816Commands, PySimCommands]:
+            cmd_set = self.find_commandsets(cmds)
+            if cmd_set:
+                self.unregister_command_set(cmd_set[0])
+
+        self.card = card
+        self.rs = rs
+
+        # When a card object and a runtime state is present, (re)equip pySim-shell with everything that is
+        # needed to operate on cards.
+        if self.card and self.rs:
+            self.lchan = self.rs.lchan[0]
+            self._onchange_conserve_write(
+                'conserve_write', False, self.conserve_write)
+            self._onchange_apdu_trace('apdu_trace', False, self.apdu_trace)
+            self.register_command_set(Iso7816Commands())
+            self.register_command_set(Ts102222Commands())
+            self.register_command_set(PySimCommands())
+            #self.iccid, sw = self.card.read_iccid()
+            self.lchan.select('MF', self)
+            rc = True
+        else:
+            self.poutput("pySim-shell not equipped!")
+
+        self.update_prompt()
+        return rc
+
+    def poutput_json(self, data, force_no_pretty=False):
+        """like cmd2.poutput() but for a JSON serializable dict."""
+        if force_no_pretty or self.json_pretty_print == False:
+            output = json.dumps(data, cls=JsonEncoder)
+        else:
+            output = json.dumps(data, cls=JsonEncoder, indent=4)
+        self.poutput(output)
+
+    def _onchange_numeric_path(self, param_name, old, new):
+        self.update_prompt()
+
+    def _onchange_conserve_write(self, param_name, old, new):
+        if self.rs:
+            self.rs.conserve_write = new
+
+    def _onchange_apdu_trace(self, param_name, old, new):
+        if self.card:
+            if new == True:
+                self.card._scc._tp.apdu_tracer = self.Cmd2ApduTracer(self)
+            else:
+                self.card._scc._tp.apdu_tracer = None
+
+    class Cmd2ApduTracer(ApduTracer):
+        def __init__(self, cmd2_app):
+            self.cmd2 = app
+
+        def trace_response(self, cmd, sw, resp):
+            self.cmd2.poutput("-> %s %s" % (cmd[:10], cmd[10:]))
+            self.cmd2.poutput("<- %s: %s" % (sw, resp))
+
+    def update_prompt(self):
+        if self.lchan:
+            path_str = self.lchan.selected_file.fully_qualified_path_str(not self.numeric_path)
+            self.prompt = 'pySIM-shell (%s)> ' % (path_str)
+        else:
+            if self.card:
+                self.prompt = 'pySIM-shell (no card profile)> '
+            else:
+                self.prompt = 'pySIM-shell (no card)> '
+
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_intro(self, _):
+        """Display the intro banner"""
+        self.poutput(self.intro)
+
+    def do_eof(self, _: argparse.Namespace) -> bool:
+        self.poutput("")
+        return self.do_quit('')
+
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_equip(self, opts):
+        """Equip pySim-shell with card"""
+        rs, card = init_card(sl)
+        self.equip(card, rs)
+
+    apdu_cmd_parser = argparse.ArgumentParser()
+    apdu_cmd_parser.add_argument('APDU', type=str, help='APDU as hex string')
+    apdu_cmd_parser.add_argument('--expect-sw', help='expect a specified status word', type=str, default=None)
+
+    @cmd2.with_argparser(apdu_cmd_parser)
+    def do_apdu(self, opts):
+        """Send a raw APDU to the card, and print SW + Response.
+        DANGEROUS: pySim-shell will not know any card state changes, and
+        not continue to work as expected if you e.g. select a different
+        file."""
+        data, sw = self.card._scc._tp.send_apdu(opts.APDU)
+        if data:
+            self.poutput("SW: %s, RESP: %s" % (sw, data))
+        else:
+            self.poutput("SW: %s" % sw)
+        if opts.expect_sw:
+            if not sw_match(sw, opts.expect_sw):
+                raise SwMatchError(sw, opts.expect_sw)
+
+    class InterceptStderr(list):
+        def __init__(self):
+            self._stderr_backup = sys.stderr
+
+        def __enter__(self):
+            self._stringio_stderr = StringIO()
+            sys.stderr = self._stringio_stderr
+            return self
+
+        def __exit__(self, *args):
+            self.stderr = self._stringio_stderr.getvalue().strip()
+            del self._stringio_stderr
+            sys.stderr = self._stderr_backup
+
+    def _show_failure_sign(self):
+        self.poutput(style("  +-------------+", fg=fg.bright_red))
+        self.poutput(style("  +   ##   ##   +", fg=fg.bright_red))
+        self.poutput(style("  +    ## ##    +", fg=fg.bright_red))
+        self.poutput(style("  +     ###     +", fg=fg.bright_red))
+        self.poutput(style("  +    ## ##    +", fg=fg.bright_red))
+        self.poutput(style("  +   ##   ##   +", fg=fg.bright_red))
+        self.poutput(style("  +-------------+", fg=fg.bright_red))
+        self.poutput("")
+
+    def _show_success_sign(self):
+        self.poutput(style("  +-------------+", fg=fg.bright_green))
+        self.poutput(style("  +          ## +", fg=fg.bright_green))
+        self.poutput(style("  +         ##  +", fg=fg.bright_green))
+        self.poutput(style("  +  #    ##    +", fg=fg.bright_green))
+        self.poutput(style("  +   ## #      +", fg=fg.bright_green))
+        self.poutput(style("  +    ##       +", fg=fg.bright_green))
+        self.poutput(style("  +-------------+", fg=fg.bright_green))
+        self.poutput("")
+
+    def _process_card(self, first, script_path):
+
+        # Early phase of card initialzation (this part may fail with an exception)
+        try:
+            rs, card = init_card(self.sl)
+            rc = self.equip(card, rs)
+        except:
+            self.poutput("")
+            self.poutput("Card initialization failed with an exception:")
+            self.poutput("---------------------8<---------------------")
+            traceback.print_exc()
+            self.poutput("---------------------8<---------------------")
+            self.poutput("")
+            return -1
+
+        # Actual card processing step. This part should never fail with an exception since the cmd2
+        # do_run_script method will catch any exception that might occur during script execution.
+        if rc:
+            self.poutput("")
+            self.poutput("Transcript stdout:")
+            self.poutput("---------------------8<---------------------")
+            with self.InterceptStderr() as logged:
+                self.do_run_script(script_path)
+            self.poutput("---------------------8<---------------------")
+
+            self.poutput("")
+            self.poutput("Transcript stderr:")
+            if logged.stderr:
+                self.poutput("---------------------8<---------------------")
+                self.poutput(logged.stderr)
+                self.poutput("---------------------8<---------------------")
+            else:
+                self.poutput("(none)")
+
+            # Check for exceptions
+            self.poutput("")
+            if "EXCEPTION of type" not in logged.stderr:
+                return 0
+
+        return -1
+
+    bulk_script_parser = argparse.ArgumentParser()
+    bulk_script_parser.add_argument(
+        'script_path', help="path to the script file")
+    bulk_script_parser.add_argument('--halt_on_error', help='stop card handling if an exeption occurs',
+                                    action='store_true')
+    bulk_script_parser.add_argument('--tries', type=int, default=2,
+                                    help='how many tries before trying the next card')
+    bulk_script_parser.add_argument('--on_stop_action', type=str, default=None,
+                                    help='commandline to execute when card handling has stopped')
+    bulk_script_parser.add_argument('--pre_card_action', type=str, default=None,
+                                    help='commandline to execute before actually talking to the card')
+
+    @cmd2.with_argparser(bulk_script_parser)
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_bulk_script(self, opts):
+        """Run script on multiple cards (bulk provisioning)"""
+
+        # Make sure that the script file exists and that it is readable.
+        if not os.access(opts.script_path, os.R_OK):
+            self.poutput("Invalid script file!")
+            return
+
+        success_count = 0
+        fail_count = 0
+
+        first = True
+        while 1:
+            # TODO: Count consecutive failures, if more than N consecutive failures occur, then stop.
+            # The ratinale is: There may be a problem with the device, we do want to prevent that
+            # all remaining cards are fired to the error bin. This is only relevant for situations
+            # with large stacks, probably we do not need this feature right now.
+
+            try:
+                # In case of failure, try multiple times.
+                for i in range(opts.tries):
+                    # fetch card into reader bay
+                    ch.get(first)
+
+                    # if necessary execute an action before we start processing the card
+                    if(opts.pre_card_action):
+                        os.system(opts.pre_card_action)
+
+                    # process the card
+                    rc = self._process_card(first, opts.script_path)
+                    if rc == 0:
+                        success_count = success_count + 1
+                        self._show_success_sign()
+                        self.poutput("Statistics: success :%i, failure: %i" % (
+                            success_count, fail_count))
+                        break
+                    else:
+                        fail_count = fail_count + 1
+                        self._show_failure_sign()
+                        self.poutput("Statistics: success :%i, failure: %i" % (
+                            success_count, fail_count))
+
+                # Depending on success or failure, the card goes either in the "error" bin or in the
+                # "done" bin.
+                if rc < 0:
+                    ch.error()
+                else:
+                    ch.done()
+
+                # In most cases it is possible to proceed with the next card, but the
+                # user may decide to halt immediately when an error occurs
+                if opts.halt_on_error and rc < 0:
+                    return
+
+            except (KeyboardInterrupt):
+                self.poutput("")
+                self.poutput("Terminated by user!")
+                return
+            except (SystemExit):
+                # When all cards are processed the card handler device will throw a SystemExit
+                # exception. Also Errors that are not recoverable (cards stuck etc.) will end up here.
+                # The user has the option to execute some action to make aware that the card handler
+                # needs service.
+                if(opts.on_stop_action):
+                    os.system(opts.on_stop_action)
+                return
+            except:
+                self.poutput("")
+                self.poutput("Card handling failed with an exception:")
+                self.poutput("---------------------8<---------------------")
+                traceback.print_exc()
+                self.poutput("---------------------8<---------------------")
+                self.poutput("")
+                fail_count = fail_count + 1
+                self._show_failure_sign()
+                self.poutput("Statistics: success :%i, failure: %i" %
+                             (success_count, fail_count))
+
+            first = False
+
+    echo_parser = argparse.ArgumentParser()
+    echo_parser.add_argument('string', help="string to echo on the shell")
+
+    @cmd2.with_argparser(echo_parser)
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_echo(self, opts):
+        """Echo (print) a string on the console"""
+        self.poutput(opts.string)
+
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_version(self, opts):
+        """Print the pySim software version."""
+        import pkg_resources
+        self.poutput(pkg_resources.get_distribution('pySim'))
+
+@with_default_category('pySim Commands')
+class PySimCommands(CommandSet):
+    def __init__(self):
+        super().__init__()
+
+    dir_parser = argparse.ArgumentParser()
+    dir_parser.add_argument(
+        '--fids', help='Show file identifiers', action='store_true')
+    dir_parser.add_argument(
+        '--names', help='Show file names', action='store_true')
+    dir_parser.add_argument(
+        '--apps', help='Show applications', action='store_true')
+    dir_parser.add_argument(
+        '--all', help='Show all selectable identifiers and names', action='store_true')
+
+    @cmd2.with_argparser(dir_parser)
+    def do_dir(self, opts):
+        """Show a listing of files available in currently selected DF or MF"""
+        if opts.all:
+            flags = []
+        elif opts.fids or opts.names or opts.apps:
+            flags = ['PARENT', 'SELF']
+            if opts.fids:
+                flags += ['FIDS', 'AIDS']
+            if opts.names:
+                flags += ['FNAMES', 'ANAMES']
+            if opts.apps:
+                flags += ['ANAMES', 'AIDS']
+        else:
+            flags = ['PARENT', 'SELF', 'FNAMES', 'ANAMES']
+        selectables = list(
+            self._cmd.lchan.selected_file.get_selectable_names(flags=flags))
+        directory_str = tabulate_str_list(
+            selectables, width=79, hspace=2, lspace=1, align_left=True)
+        path = self._cmd.lchan.selected_file.fully_qualified_path_str(True)
+        self._cmd.poutput(path)
+        path = self._cmd.lchan.selected_file.fully_qualified_path_str(False)
+        self._cmd.poutput(path)
+        self._cmd.poutput(directory_str)
+        self._cmd.poutput("%d files" % len(selectables))
+
+    def walk(self, indent=0, action_ef=None, action_df=None, context=None, **kwargs):
+        """Recursively walk through the file system, starting at the currently selected DF"""
+
+        if isinstance(self._cmd.lchan.selected_file, CardDF):
+            if action_df:
+                action_df(context, opts)
+
+        files = self._cmd.lchan.selected_file.get_selectables(
+            flags=['FNAMES', 'ANAMES'])
+        for f in files:
+            # special case: When no action is performed, just output a directory
+            if not action_ef and not action_df:
+                output_str = "  " * indent + str(f) + (" " * 250)
+                output_str = output_str[0:25]
+                if isinstance(files[f], CardADF):
+                    output_str += " " + str(files[f].aid)
+                else:
+                    output_str += " " + str(files[f].fid)
+                output_str += " " + str(files[f].desc)
+                self._cmd.poutput(output_str)
+
+            if isinstance(files[f], CardDF):
+                skip_df = False
+                try:
+                    fcp_dec = self._cmd.lchan.select(f, self._cmd)
+                except Exception as e:
+                    skip_df = True
+                    df = self._cmd.lchan.selected_file
+                    df_path = df.fully_qualified_path_str(True)
+                    df_skip_reason_str = df_path + \
+                        "/" + str(f) + ", " + str(e)
+                    if context:
+                        context['DF_SKIP'] += 1
+                        context['DF_SKIP_REASON'].append(df_skip_reason_str)
+
+                # If the DF was skipped, we never have entered the directory
+                # below, so we must not move up.
+                if skip_df == False:
+                    self.walk(indent + 1, action_ef, action_df, context, **kwargs)
+                    fcp_dec = self._cmd.lchan.select("..", self._cmd)
+
+            elif action_ef:
+                df_before_action = self._cmd.lchan.selected_file
+                action_ef(f, context, **kwargs)
+                # When walking through the file system tree the action must not
+                # always restore the currently selected file to the file that
+                # was selected before executing the action() callback.
+                if df_before_action != self._cmd.lchan.selected_file:
+                    raise RuntimeError("inconsistent walk, %s is currently selected but expecting %s to be selected"
+                                       % (str(self._cmd.lchan.selected_file), str(df_before_action)))
+
+    def do_tree(self, opts):
+        """Display a filesystem-tree with all selectable files"""
+        self.walk()
+
+    def export_ef(self, filename, context, as_json):
+        """ Select and export a single elementary file (EF) """
+        context['COUNT'] += 1
+        df = self._cmd.lchan.selected_file
+
+	# The currently selected file (not the file we are going to export)
+	# must always be an ADF or DF. From this starting point we select
+	# the EF we want to export. To maintain consistency we will then
+	# select the current DF again (see comment below).
+        if not isinstance(df, CardDF):
+            raise RuntimeError(
+                "currently selected file %s is not a DF or ADF" % str(df))
+
+        df_path_list = df.fully_qualified_path(True)
+        df_path = df.fully_qualified_path_str(True)
+        df_path_fid = df.fully_qualified_path_str(False)
+
+        file_str = df_path + "/" + str(filename)
+        self._cmd.poutput(boxed_heading_str(file_str))
+
+        self._cmd.poutput("# directory: %s (%s)" % (df_path, df_path_fid))
+        try:
+            fcp_dec = self._cmd.lchan.select(filename, self._cmd)
+            self._cmd.poutput("# file: %s (%s)" % (
+                self._cmd.lchan.selected_file.name, self._cmd.lchan.selected_file.fid))
+
+            structure = self._cmd.lchan.selected_file_structure()
+            self._cmd.poutput("# structure: %s" % str(structure))
+            self._cmd.poutput("# RAW FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp_hex))
+            self._cmd.poutput("# Decoded FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp))
+
+            for f in df_path_list:
+                self._cmd.poutput("select " + str(f))
+            self._cmd.poutput("select " + self._cmd.lchan.selected_file.name)
+
+            if structure == 'transparent':
+                if as_json:
+                    result = self._cmd.lchan.read_binary_dec()
+                    self._cmd.poutput("update_binary_decoded '%s'" % json.dumps(result[0], cls=JsonEncoder))
+                else:
+                    result = self._cmd.lchan.read_binary()
+                    self._cmd.poutput("update_binary " + str(result[0]))
+            elif structure == 'cyclic' or structure == 'linear_fixed':
+                # Use number of records specified in select response
+                num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
+                if num_of_rec:
+                    for r in range(1, num_of_rec + 1):
+                        if as_json:
+                            result = self._cmd.lchan.read_record_dec(r)
+                            self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
+                        else:
+                            result = self._cmd.lchan.read_record(r)
+                            self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
+
+                # When the select response does not return the number of records, read until we hit the
+                # first record that cannot be read.
+                else:
+                    r = 1
+                    while True:
+                        try:
+                            if as_json:
+                                result = self._cmd.lchan.read_record_dec(r)
+                                self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
+                            else:
+                                result = self._cmd.lchan.read_record(r)
+                                self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
+                        except SwMatchError as e:
+                            # We are past the last valid record - stop
+                            if e.sw_actual == "9402":
+                                break
+                            # Some other problem occurred
+                            else:
+                                raise e
+                        r = r + 1
+            elif structure == 'ber_tlv':
+                tags = self._cmd.lchan.retrieve_tags()
+                for t in tags:
+                    result = self._cmd.lchan.retrieve_data(t)
+                    (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
+                    self._cmd.poutput("set_data 0x%02x %s" % (t, b2h(val)))
+            else:
+                raise RuntimeError(
+                    'Unsupported structure "%s" of file "%s"' % (structure, filename))
+        except Exception as e:
+            bad_file_str = df_path + "/" + str(filename) + ", " + str(e)
+            self._cmd.poutput("# bad file: %s" % bad_file_str)
+            context['ERR'] += 1
+            context['BAD'].append(bad_file_str)
+
+        # When reading the file is done, make sure the parent file is
+        # selected again. This will be the usual case, however we need
+        # to check before since we must not select the same DF twice
+        if df != self._cmd.lchan.selected_file:
+            self._cmd.lchan.select(df.fid or df.aid, self._cmd)
+
+        self._cmd.poutput("#")
+
+    export_parser = argparse.ArgumentParser()
+    export_parser.add_argument(
+        '--filename', type=str, default=None, help='only export specific file')
+    export_parser.add_argument(
+        '--json', action='store_true', help='export as JSON (less reliable)')
+
+    @cmd2.with_argparser(export_parser)
+    def do_export(self, opts):
+        """Export files to script that can be imported back later"""
+        context = {'ERR': 0, 'COUNT': 0, 'BAD': [],
+                   'DF_SKIP': 0, 'DF_SKIP_REASON': []}
+        kwargs_export = {'as_json': opts.json}
+        exception_str_add = ""
+
+        if opts.filename:
+            self.export_ef(opts.filename, context, **kwargs_export)
+        else:
+            try:
+                self.walk(0, self.export_ef, None, context, **kwargs_export)
+            except Exception as e:
+                print("# Stopping early here due to exception: " + str(e))
+                print("#")
+                exception_str_add = ", also had to stop early due to exception:" + str(e)
+
+        self._cmd.poutput(boxed_heading_str("Export summary"))
+
+        self._cmd.poutput("# total files visited: %u" % context['COUNT'])
+        self._cmd.poutput("# bad files:           %u" % context['ERR'])
+        for b in context['BAD']:
+            self._cmd.poutput("#  " + b)
+
+        self._cmd.poutput("# skipped dedicated files(s): %u" %
+                          context['DF_SKIP'])
+        for b in context['DF_SKIP_REASON']:
+            self._cmd.poutput("#  " + b)
+
+        if context['ERR'] and context['DF_SKIP']:
+            raise RuntimeError("unable to export %i elementary file(s) and %i dedicated file(s)%s" % (
+                    context['ERR'], context['DF_SKIP'], exception_str_add))
+        elif context['ERR']:
+            raise RuntimeError(
+                    "unable to export %i elementary file(s)%s" % (context['ERR'], exception_str_add))
+        elif context['DF_SKIP']:
+            raise RuntimeError(
+                    "unable to export %i dedicated files(s)%s" % (context['ERR'], exception_str_add))
+
+    def do_reset(self, opts):
+        """Reset the Card."""
+        atr = self._cmd.lchan.reset(self._cmd)
+        self._cmd.poutput('Card ATR: %s' % atr)
+        self._cmd.update_prompt()
+
+    def do_desc(self, opts):
+        """Display human readable file description for the currently selected file"""
+        desc = self._cmd.lchan.selected_file.desc
+        if desc:
+            self._cmd.poutput(desc)
+        else:
+            self._cmd.poutput("no description available")
+
+    def do_verify_adm(self, arg):
+        """VERIFY the ADM1 PIN"""
+        if arg:
+            # use specified ADM-PIN
+            pin_adm = sanitize_pin_adm(arg)
+        else:
+            # try to find an ADM-PIN if none is specified
+            result = card_key_provider_get_field(
+                'ADM1', key='ICCID', value=self._cmd.iccid)
+            pin_adm = sanitize_pin_adm(result)
+            if pin_adm:
+                self._cmd.poutput(
+                    "found ADM-PIN '%s' for ICCID '%s'" % (result, self._cmd.iccid))
+            else:
+                raise ValueError(
+                    "cannot find ADM-PIN for ICCID '%s'" % (self._cmd.iccid))
+
+        if pin_adm:
+            self._cmd.card.verify_adm(h2b(pin_adm))
+        else:
+            raise ValueError("error: cannot authenticate, no adm-pin!")
+
+
+@with_default_category('ISO7816 Commands')
+class Iso7816Commands(CommandSet):
+    def __init__(self):
+        super().__init__()
+
+    def do_select(self, opts):
+        """SELECT a File (ADF/DF/EF)"""
+        if len(opts.arg_list) == 0:
+            path = self._cmd.lchan.selected_file.fully_qualified_path_str(True)
+            path_fid = self._cmd.lchan.selected_file.fully_qualified_path_str(False)
+            self._cmd.poutput("currently selected file: %s (%s)" % (path, path_fid))
+            return
+
+        path = opts.arg_list[0]
+        fcp_dec = self._cmd.lchan.select(path, self._cmd)
+        self._cmd.update_prompt()
+        self._cmd.poutput_json(fcp_dec)
+
+    def complete_select(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for SELECT"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    def get_code(self, code):
+        """Use code either directly or try to get it from external data source"""
+        auto = ('PIN1', 'PIN2', 'PUK1', 'PUK2')
+
+        if str(code).upper() not in auto:
+            return sanitize_pin_adm(code)
+
+        result = card_key_provider_get_field(
+            str(code), key='ICCID', value=self._cmd.iccid)
+        result = sanitize_pin_adm(result)
+        if result:
+            self._cmd.poutput("found %s '%s' for ICCID '%s'" %
+                              (code.upper(), result, self._cmd.iccid))
+        else:
+            self._cmd.poutput("cannot find %s for ICCID '%s'" %
+                              (code.upper(), self._cmd.iccid))
+        return result
+
+    verify_chv_parser = argparse.ArgumentParser()
+    verify_chv_parser.add_argument(
+        '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
+    verify_chv_parser.add_argument(
+        'pin_code', type=str, help='PIN code digits, \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+
+    @cmd2.with_argparser(verify_chv_parser)
+    def do_verify_chv(self, opts):
+        """Verify (authenticate) using specified CHV (PIN) code, which is how the specifications
+        call it if you authenticate yourself using the specified PIN.  There usually is at least PIN1 and
+        PIN2."""
+        pin = self.get_code(opts.pin_code)
+        (data, sw) = self._cmd.card._scc.verify_chv(opts.pin_nr, h2b(pin))
+        self._cmd.poutput("CHV verification successful")
+
+    unblock_chv_parser = argparse.ArgumentParser()
+    unblock_chv_parser.add_argument(
+        '--pin-nr', type=int, default=1, help='PUK Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
+    unblock_chv_parser.add_argument(
+        'puk_code', type=str, help='PUK code digits \"PUK1\" or \"PUK2\" to get PUK code from external data source')
+    unblock_chv_parser.add_argument(
+        'new_pin_code', type=str, help='PIN code digits \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+
+    @cmd2.with_argparser(unblock_chv_parser)
+    def do_unblock_chv(self, opts):
+        """Unblock PIN code using specified PUK code"""
+        new_pin = self.get_code(opts.new_pin_code)
+        puk = self.get_code(opts.puk_code)
+        (data, sw) = self._cmd.card._scc.unblock_chv(
+            opts.pin_nr, h2b(puk), h2b(new_pin))
+        self._cmd.poutput("CHV unblock successful")
+
+    change_chv_parser = argparse.ArgumentParser()
+    change_chv_parser.add_argument(
+        '--pin-nr', type=int, default=1, help='PUK Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
+    change_chv_parser.add_argument(
+        'pin_code', type=str, help='PIN code digits \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+    change_chv_parser.add_argument(
+        'new_pin_code', type=str, help='PIN code digits \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+
+    @cmd2.with_argparser(change_chv_parser)
+    def do_change_chv(self, opts):
+        """Change PIN code to a new PIN code"""
+        new_pin = self.get_code(opts.new_pin_code)
+        pin = self.get_code(opts.pin_code)
+        (data, sw) = self._cmd.card._scc.change_chv(
+            opts.pin_nr, h2b(pin), h2b(new_pin))
+        self._cmd.poutput("CHV change successful")
+
+    disable_chv_parser = argparse.ArgumentParser()
+    disable_chv_parser.add_argument(
+        '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
+    disable_chv_parser.add_argument(
+        'pin_code', type=str, help='PIN code digits, \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+
+    @cmd2.with_argparser(disable_chv_parser)
+    def do_disable_chv(self, opts):
+        """Disable PIN code using specified PIN code"""
+        pin = self.get_code(opts.pin_code)
+        (data, sw) = self._cmd.card._scc.disable_chv(opts.pin_nr, h2b(pin))
+        self._cmd.poutput("CHV disable successful")
+
+    enable_chv_parser = argparse.ArgumentParser()
+    enable_chv_parser.add_argument(
+        '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
+    enable_chv_parser.add_argument(
+        'pin_code', type=str, help='PIN code digits, \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+
+    @cmd2.with_argparser(enable_chv_parser)
+    def do_enable_chv(self, opts):
+        """Enable PIN code using specified PIN code"""
+        pin = self.get_code(opts.pin_code)
+        (data, sw) = self._cmd.card._scc.enable_chv(opts.pin_nr, h2b(pin))
+        self._cmd.poutput("CHV enable successful")
+
+    def do_deactivate_file(self, opts):
+        """Deactivate the currently selected EF"""
+        (data, sw) = self._cmd.card._scc.deactivate_file()
+
+    activate_file_parser = argparse.ArgumentParser()
+    activate_file_parser.add_argument('NAME', type=str, help='File name or FID of file to activate')
+    @cmd2.with_argparser(activate_file_parser)
+    def do_activate_file(self, opts):
+        """Activate the specified EF. This used to be called REHABILITATE in TS 11.11 for classic
+        SIM.  You need to specify the name or FID of the file to activate."""
+        (data, sw) = self._cmd.lchan.activate_file(opts.NAME)
+
+    def complete_activate_file(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for ACTIVATE FILE"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    open_chan_parser = argparse.ArgumentParser()
+    open_chan_parser.add_argument(
+        'chan_nr', type=int, default=0, help='Channel Number')
+
+    @cmd2.with_argparser(open_chan_parser)
+    def do_open_channel(self, opts):
+        """Open a logical channel."""
+        (data, sw) = self._cmd.card._scc.manage_channel(
+            mode='open', lchan_nr=opts.chan_nr)
+
+    close_chan_parser = argparse.ArgumentParser()
+    close_chan_parser.add_argument(
+        'chan_nr', type=int, default=0, help='Channel Number')
+
+    @cmd2.with_argparser(close_chan_parser)
+    def do_close_channel(self, opts):
+        """Close a logical channel."""
+        (data, sw) = self._cmd.card._scc.manage_channel(
+            mode='close', lchan_nr=opts.chan_nr)
+
+    def do_status(self, opts):
+        """Perform the STATUS command."""
+        fcp_dec = self._cmd.lchan.status()
+        self._cmd.poutput_json(fcp_dec)
+
+    suspend_uicc_parser = argparse.ArgumentParser()
+    suspend_uicc_parser.add_argument('--min-duration-secs', type=int, default=60,
+                                     help='Proposed minimum duration of suspension')
+    suspend_uicc_parser.add_argument('--max-duration-secs', type=int, default=24*60*60,
+                                     help='Proposed maximum duration of suspension')
+
+    # not ISO7816-4 but TS 102 221
+    @cmd2.with_argparser(suspend_uicc_parser)
+    def do_suspend_uicc(self, opts):
+        """Perform the SUSPEND UICC command. Only supported on some UICC."""
+        (duration, token, sw) = self._cmd.card._scc.suspend_uicc(min_len_secs=opts.min_duration_secs,
+                                                                 max_len_secs=opts.max_duration_secs)
+        self._cmd.poutput(
+            'Negotiated Duration: %u secs, Token: %s, SW: %s' % (duration, token, sw))
+
+class Proact(ProactiveHandler):
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        # print its parsed representation
+        print(pcmd.decoded)
+        # TODO: implement the basics, such as SMS Sending, ...
+
+
+
+option_parser = argparse.ArgumentParser(prog='pySim-shell', description='interactive SIM card shell',
+                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+argparse_add_reader_args(option_parser)
+
+global_group = option_parser.add_argument_group('General Options')
+global_group.add_argument('--script', metavar='PATH', default=None,
+                          help='script with pySim-shell commands to be executed automatically at start-up')
+global_group.add_argument('--csv', metavar='FILE',
+                          default=None, help='Read card data from CSV file')
+global_group.add_argument("--card_handler", dest="card_handler_config", metavar="FILE",
+                          help="Use automatic card handling machine")
+
+adm_group = global_group.add_mutually_exclusive_group()
+adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', default=None,
+                       help='ADM PIN used for provisioning (overwrites default)')
+adm_group.add_argument('-A', '--pin-adm-hex', metavar='PIN_ADM1_HEX', dest='pin_adm_hex', default=None,
+                       help='ADM PIN used for provisioning, as hex string (16 characters long)')
+
+
+if __name__ == '__main__':
+
+    # Parse options
+    opts = option_parser.parse_args()
+
+    # If a script file is specified, be sure that it actually exists
+    if opts.script:
+        if not os.access(opts.script, os.R_OK):
+            print("Invalid script file!")
+            sys.exit(2)
+
+    # Register csv-file as card data provider, either from specified CSV
+    # or from CSV file in home directory
+    csv_default = str(Path.home()) + "/.osmocom/pysim/card_data.csv"
+    if opts.csv:
+        card_key_provider_register(CardKeyProviderCsv(opts.csv))
+    if os.path.isfile(csv_default):
+        card_key_provider_register(CardKeyProviderCsv(csv_default))
+
+    # Init card reader driver
+    sl = init_reader(opts, proactive_handler = Proact())
+    if sl is None:
+        exit(1)
+
+    # Create command layer
+    scc = SimCardCommands(transport=sl)
+
+    # Create a card handler (for bulk provisioning)
+    if opts.card_handler_config:
+        ch = CardHandlerAuto(None, opts.card_handler_config)
+    else:
+        ch = CardHandler(sl)
+
+    # Detect and initialize the card in the reader. This may fail when there
+    # is no card in the reader or the card is unresponsive. PysimApp is
+    # able to tolerate and recover from that.
+    try:
+        rs, card = init_card(sl)
+        app = PysimApp(card, rs, sl, ch, opts.script)
+    except:
+        print("Card initialization failed with an exception:")
+        print("---------------------8<---------------------")
+        traceback.print_exc()
+        print("---------------------8<---------------------")
+        print("(you may still try to recover from this manually by using the 'equip' command.)")
+        print(
+            " it should also be noted that some readers may behave strangely when no card")
+        print(" is inserted.)")
+        print("")
+        app = PysimApp(card, None, sl, ch, opts.script)
+
+    # If the user supplies an ADM PIN at via commandline args authenticate
+    # immediately so that the user does not have to use the shell commands
+    pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
+    if pin_adm:
+        if not card:
+            print("Card error, cannot do ADM verification with supplied ADM pin now.")
+        try:
+            card.verify_adm(h2b(pin_adm))
+        except Exception as e:
+            print(e)
+
+    app.cmdloop()

pySim-trace.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+
+import sys
+import logging, colorlog
+import argparse
+from pprint import pprint as pp
+
+from pySim.apdu import *
+from pySim.filesystem import RuntimeState
+
+from pySim.cards import UsimCard
+from pySim.commands import SimCardCommands
+from pySim.profile import CardProfile
+from pySim.ts_102_221 import CardProfileUICCSIM
+from pySim.ts_31_102 import CardApplicationUSIM
+from pySim.ts_31_103 import CardApplicationISIM
+from pySim.transport import LinkBase
+
+from pySim.apdu_source.gsmtap import GsmtapApduSource
+from pySim.apdu_source.pyshark_rspro import PysharkRsproPcap, PysharkRsproLive
+
+from pySim.apdu.ts_102_221 import UiccSelect, UiccStatus
+
+log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+colorlog.basicConfig(level=logging.INFO, format = log_format)
+logger = colorlog.getLogger()
+
+# merge all of the command sets into one global set. This will override instructions,
+# the one from the 'last' set in the addition below will prevail.
+from pySim.apdu.ts_51_011 import ApduCommands as SimApduCommands
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+ApduCommands = SimApduCommands + UiccApduCommands + UsimApduCommands #+ GpApduCommands
+
+
+class DummySimLink(LinkBase):
+    """A dummy implementation of the LinkBase abstract base class.  Currently required
+    as the UsimCard doesn't work without SimCardCommands, which in turn require
+    a LinkBase implementation talking to a card.
+
+    In the tracer, we don't actually talk to any card, so we simply drop everything
+    and claim it is successful.
+
+    The UsimCard / SimCardCommands should be refactored to make this obsolete later."""
+    def __init__(self, debug: bool = False, **kwargs):
+        super().__init__(**kwargs)
+        self._debug = debug
+        self._atr = h2i('3B9F96801F878031E073FE211B674A4C753034054BA9')
+
+    def _send_apdu_raw(self, pdu):
+        #print("DummySimLink-apdu: %s" % pdu)
+        return [], '9000'
+
+    def connect(self):
+        pass
+
+    def disconnect(self):
+        pass
+
+    def reset_card(self):
+        return 1
+
+    def get_atr(self):
+        return self._atr
+
+    def wait_for_card(self):
+        pass
+
+
+class Tracer:
+    def __init__(self, **kwargs):
+        # we assume a generic SIM + UICC + USIM + ISIM card
+        profile = CardProfileUICCSIM()
+        profile.add_application(CardApplicationUSIM())
+        profile.add_application(CardApplicationISIM())
+        scc = SimCardCommands(transport=DummySimLink())
+        card = UsimCard(scc)
+        self.rs = RuntimeState(card, profile)
+        # APDU Decoder
+        self.ad = ApduDecoder(ApduCommands)
+        # parameters
+        self.suppress_status = kwargs.get('suppress_status', True)
+        self.suppress_select = kwargs.get('suppress_select', True)
+        self.source = kwargs.get('source', None)
+
+    def format_capdu(self, inst: ApduCommand):
+        """Output a single decoded + processed ApduCommand."""
+        print("%02u %-16s %-35s %-8s %s %s" % (inst.lchan_nr, inst._name, inst.path_str, inst.col_id, inst.col_sw, inst.processed))
+        print("===============================")
+
+    def main(self):
+        """Main loop of tracer: Iterates over all Apdu received from source."""
+        while True:
+            # obtain the next APDU from the source (blocking read)
+            apdu = self.source.read()
+            #print(apdu)
+
+            if isinstance(apdu, CardReset):
+                self.rs.reset()
+                continue
+
+            # ask ApduDecoder to look-up (INS,CLA) + instantiate an ApduCommand derived
+            # class like 'UiccSelect'
+            inst = self.ad.input(apdu)
+            # process the APDU (may modify the RuntimeState)
+            inst.process(self.rs)
+
+            # Avoid cluttering the log with too much verbosity
+            if self.suppress_select and isinstance(inst, UiccSelect):
+                continue
+            if self.suppress_status and isinstance(inst, UiccStatus):
+                continue
+            #print(inst)
+            self.format_capdu(inst)
+
+option_parser = argparse.ArgumentParser(prog='pySim-trace', description='Osmocom pySim high-level SIM card trace decoder',
+                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+global_group = option_parser.add_argument_group('General Options')
+global_group.add_argument('--no-suppress-select', action='store_false', dest='suppress_select',
+                          help="Don't suppress displaying SELECT APDUs")
+global_group.add_argument('--no-suppress-status', action='store_false', dest='suppress_status',
+                          help="Don't suppress displaying STATUS APDUs")
+
+subparsers = option_parser.add_subparsers(help='APDU Source', dest='source', required=True)
+
+parser_gsmtap = subparsers.add_parser('gsmtap-udp', help='Live capture of GSMTAP-SIM on UDP port')
+parser_gsmtap.add_argument('-i', '--bind-ip', default='127.0.0.1',
+                           help='Local IP address to which to bind the UDP port')
+parser_gsmtap.add_argument('-p', '--bind-port', default=4729,
+                           help='Local UDP port')
+
+parser_rspro_pyshark_pcap = subparsers.add_parser('rspro-pyshark-pcap', help="""
+    PCAP file containing RSPRO (osmo-remsim) communication; processed via pyshark.
+    REQUIRES OSMOCOM PATCHED WIRESHARK!""")
+parser_rspro_pyshark_pcap.add_argument('-f', '--pcap-file', required=True,
+                                       help='Name of the PCAP[ng] file to be read')
+
+parser_rspro_pyshark_live = subparsers.add_parser('rspro-pyshark-live', help="""
+    Live capture of RSPRO (osmo-remsim) communication; processed via pyshark.
+    REQUIRES OSMOCOM PATCHED WIRESHARK!""")
+parser_rspro_pyshark_live.add_argument('-i', '--interface', required=True,
+                                       help='Name of the network interface to capture on')
+
+if __name__ == '__main__':
+
+    opts = option_parser.parse_args()
+
+    logger.info('Opening source %s...' % opts.source)
+    if opts.source == 'gsmtap-udp':
+        s = GsmtapApduSource(opts.bind_ip, opts.bind_port)
+    elif opts.source == 'rspro-pyshark-pcap':
+        s = PysharkRsproPcap(opts.pcap_file)
+    elif opts.source == 'rspro-pyshark-live':
+        s = PysharkRsproLive(opts.interface)
+
+    tracer = Tracer(source=s, suppress_status=opts.suppress_status, suppress_select=opts.suppress_select)
+    logger.info('Entering main loop...')
+    tracer.main()
+

pySim/apdu/__init__.py

@@ -0,0 +1,446 @@
+# coding=utf-8
+"""APDU (and TPDU) parser for UICC/USIM/ISIM cards.
+
+The File (and its classes) represent the structure / hierarchy
+of the APDUs as seen in SIM/UICC/SIM/ISIM cards.  The primary use case
+is to perform a meaningful decode of protocol traces taken between card and UE.
+
+The ancient wirshark dissector developed for GSMTAP generated by SIMtrace
+is far too simplistic, while this decoder can utilize all of the information
+we already know in pySim about the filesystem structure, file encoding, etc.
+"""
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import abc
+from termcolor import colored
+import typing
+from typing import List, Dict, Optional
+
+from construct import *
+from construct import Optional as COptional
+from pySim.construct import *
+from pySim.utils import *
+from pySim.filesystem import RuntimeLchan, RuntimeState, lchan_nr_from_cla
+from pySim.filesystem import CardADF, CardFile, TransparentEF, LinFixedEF
+
+"""There are multiple levels of decode:
+
+    1) pure TPDU / APDU level (no filesystem state required to decode)
+        1a) the raw C-TPDU + R-TPDU
+        1b) the raw C-APDU + R-APDU
+        1c) the C-APDU + R-APDU split in its portions (p1/p2/lc/le/cmd/rsp)
+        1d) the abstract C-APDU + R-APDU (mostly p1/p2 parsing; SELECT response)
+    2) the decoded DATA of command/response APDU
+        * READ/UPDATE: requires state/context: which file is selected? how to decode it?
+"""
+
+class ApduCommandMeta(abc.ABCMeta):
+    """A meta-class that we can use to set some class variables when declaring
+       a derived class of ApduCommand."""
+    def __new__(metacls, name, bases, namespace, **kwargs):
+        x = super().__new__(metacls, name, bases, namespace)
+        x._name = namespace.get('name', kwargs.get('n', None))
+        x._ins = namespace.get('ins', kwargs.get('ins', None))
+        x._cla = namespace.get('cla', kwargs.get('cla', None))
+        return x
+
+BytesOrHex = typing.Union[bytes, Hexstr]
+
+class Tpdu:
+    def __init__(self, cmd: BytesOrHex, rsp: Optional[BytesOrHex] = None):
+        if isinstance(cmd, str):
+            self.cmd = h2b(cmd)
+        else:
+            self.cmd = cmd
+        if  isinstance(rsp, str):
+            self.rsp = h2b(rsp)
+        else:
+            self.rsp = rsp
+
+    def __str__(self):
+        return '%s(%02X %02X %02X %02X %02X %s %s %s)' % (type(self).__name__, self.cla, self.ins, self.p1,
+                self.p2, self.p3, b2h(self.cmd_data), b2h(self.rsp_data), b2h(self.sw))
+
+    @property
+    def cla(self) -> int:
+        """Return CLA of the C-APDU Header."""
+        return self.cmd[0]
+
+    @property
+    def ins(self) -> int:
+        """Return INS of the C-APDU Header."""
+        return self.cmd[1]
+
+    @property
+    def p1(self) -> int:
+        """Return P1 of the C-APDU Header."""
+        return self.cmd[2]
+
+    @property
+    def p2(self) -> int:
+        """Return P2 of the C-APDU Header."""
+        return self.cmd[3]
+
+    @property
+    def p3(self) -> int:
+        """Return P3 of the C-APDU Header."""
+        return self.cmd[4]
+
+    @property
+    def cmd_data(self) -> int:
+        """Return the DATA portion of the C-APDU"""
+        return self.cmd[5:]
+
+    @property
+    def sw(self) -> Optional[bytes]:
+        """Return Status Word (SW) of the R-APDU"""
+        return self.rsp[-2:] if self.rsp else None
+
+    @property
+    def rsp_data(self) -> Optional[bytes]:
+        """Return the DATA portion of the R-APDU"""
+        return self.rsp[:-2] if self.rsp else None
+
+
+class Apdu(Tpdu):
+    @property
+    def lc(self) -> int:
+        """Return Lc; Length of C-APDU body."""
+        return len(self.cmd_data)
+
+    @property
+    def lr(self) -> int:
+        """Return Lr; Length of R-APDU body."""
+        return len(self.rsp_data)
+
+    @property
+    def successful(self) -> bool:
+        """Was the execution of this APDU successful?"""
+        method = getattr(self, '_is_success', None)
+        if callable(method):
+            return method()
+        # default case: only 9000 is success
+        return self.sw == b'\x90\x00'
+
+
+class ApduCommand(Apdu, metaclass=ApduCommandMeta):
+    """Base class from which you would derive individual commands/instructions like SELECT.
+       A derived class represents a decoder for a specific instruction.
+       An instance of such a derived class is one concrete APDU."""
+    # fall-back constructs if the derived class provides no override
+    _construct_p1 = Byte
+    _construct_p2 = Byte
+    _construct = HexAdapter(GreedyBytes)
+    _construct_rsp = HexAdapter(GreedyBytes)
+
+    def __init__(self, cmd: BytesOrHex, rsp: Optional[BytesOrHex] = None):
+        """Instantiate a new ApduCommand from give cmd + resp."""
+        # store raw data
+        super().__init__(cmd, rsp)
+        # default to 'empty' ID column. To be set to useful values (like record number)
+        # by derived class {cmd_rsp}_to_dict() or process() methods
+        self.col_id = '-'
+        # fields only set by process_* methods
+        self.file = None
+        self.lchan = None
+        self.processed = None
+        # the methods below could raise exceptions and those handlers might assume cmd_{dict,resp}
+        self.cmd_dict = None
+        self.rsp_dict = None
+        # interpret the data
+        self.cmd_dict = self.cmd_to_dict()
+        self.rsp_dict = self.rsp_to_dict() if self.rsp else {}
+
+
+    @classmethod
+    def from_apdu(cls, apdu:Apdu, **kwargs) -> 'ApduCommand':
+        """Instantiate an ApduCommand from an existing APDU."""
+        return cls(cmd=apdu.cmd, rsp=apdu.rsp, **kwargs)
+
+    @classmethod
+    def from_bytes(cls, buffer:bytes) -> 'ApduCommand':
+        """Instantiate an ApduCommand from a linear byte buffer containing hdr,cmd,rsp,sw.
+        This is for example used when parsing GSMTAP traces that traditionally contain the
+        full command and response portion in one packet: "CLA INS P1 P2 P3 DATA SW" and we
+        now need to figure out whether the DATA part is part of the CMD or the RSP"""
+        apdu_case = cls.get_apdu_case(buffer)
+        if apdu_case in [1, 2]:
+            # data is part of response
+            return cls(buffer[:5], buffer[5:])
+        elif apdu_case in [3, 4]:
+            # data is part of command
+            lc = buffer[4]
+            return cls(buffer[:5+lc], buffer[5+lc:])
+        else:
+            raise ValueError('%s: Invalid APDU Case %u' % (cls.__name__, apdu_case))
+
+    @property
+    def path(self) -> List[str]:
+        """Return (if known) the path as list of files to the file on which this command operates."""
+        if self.file:
+            return self.file.fully_qualified_path()
+        else:
+            return []
+
+    @property
+    def path_str(self) -> str:
+        """Return (if known) the path as string to the file on which this command operates."""
+        if self.file:
+            return self.file.fully_qualified_path_str()
+        else:
+            return ''
+
+    @property
+    def col_sw(self) -> str:
+        """Return the ansi-colorized status word. Green==OK, Red==Error"""
+        if self.successful:
+            return colored(b2h(self.sw), 'green')
+        else:
+            return colored(b2h(self.sw), 'red')
+
+    @property
+    def lchan_nr(self) -> int:
+        """Logical channel number over which this ApduCommand was transmitted."""
+        if self.lchan:
+            return self.lchan.lchan_nr
+        else:
+            return lchan_nr_from_cla(self.cla)
+
+    def __str__(self) -> str:
+        return '%02u %s(%s): %s' % (self.lchan_nr, type(self).__name__, self.path_str, self.to_dict())
+
+    def __repr__(self) -> str:
+        return '%s(INS=%02x,CLA=%s)' % (self.__class__, self.ins, self.cla)
+
+    def _process_fallback(self, rs: RuntimeState):
+        """Fall-back function to be called if there is no derived-class-specific
+        process_global or process_on_lchan method. Uses information from APDU decode."""
+        self.processed = {}
+        if not 'p1' in self.cmd_dict:
+            self.processed = self.to_dict()
+        else:
+            self.processed['p1'] = self.cmd_dict['p1']
+            self.processed['p2'] = self.cmd_dict['p2']
+            if 'body' in self.cmd_dict and self.cmd_dict['body']:
+                self.processed['cmd'] = self.cmd_dict['body']
+            if 'body' in self.rsp_dict and self.rsp_dict['body']:
+                self.processed['rsp'] = self.rsp_dict['body']
+        return self.processed
+
+    def process(self, rs: RuntimeState):
+        # if there is a global method, use that; else use process_on_lchan
+        method = getattr(self, 'process_global', None)
+        if callable(method):
+            self.processed = method(rs)
+            return self.processed
+        method = getattr(self, 'process_on_lchan', None)
+        if callable(method):
+            self.lchan = rs.get_lchan_by_cla(self.cla)
+            self.processed = method(self.lchan)
+            return self.processed
+        # if none of the two methods exist:
+        return self._process_fallback(rs)
+
+    @classmethod
+    def get_apdu_case(cls, hdr:bytes) -> int:
+        if hasattr(cls, '_apdu_case'):
+            return cls._apdu_case
+        method = getattr(cls, '_get_apdu_case', None)
+        if callable(method):
+            return method(hdr)
+        raise ValueError('%s: Class definition missing _apdu_case attribute or _get_apdu_case method' % cls.__name__)
+
+    @classmethod
+    def match_cla(cls, cla) -> bool:
+        """Does the given CLA match the CLA list of the command?."""
+        if not isinstance(cla, str):
+            cla = '%02X' % cla
+        cla = cla.lower()
+        # see https://github.com/PyCQA/pylint/issues/7219
+        # pylint: disable=no-member
+        for cla_match in cls._cla:
+            cla_masked = ""
+            for i in range(0, 2):
+                if cla_match[i] == 'X':
+                    cla_masked += 'X'
+                else:
+                    cla_masked += cla[i]
+            if cla_masked == cla_match:
+                return True
+        return False
+
+    def cmd_to_dict(self) -> Dict:
+        """Convert the Command part of the APDU to a dict."""
+        method = getattr(self, '_decode_cmd', None)
+        if callable(method):
+            return method()
+        else:
+            r = {}
+            method = getattr(self, '_decode_p1p2', None)
+            if callable(method):
+                r = self._decode_p1p2()
+            else:
+                r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
+                r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
+            r['p3'] = self.p3
+            if self.cmd_data:
+                r['body'] = parse_construct(self._construct, self.cmd_data)
+            return r
+
+    def rsp_to_dict(self) -> Dict:
+        """Convert the Response part of the APDU to a dict."""
+        method = getattr(self, '_decode_rsp', None)
+        if callable(method):
+            return method()
+        else:
+            r = {}
+            if self.rsp_data:
+                r['body'] = parse_construct(self._construct_rsp, self.rsp_data)
+            r['sw'] = b2h(self.sw)
+            return r
+
+    def to_dict(self) -> Dict:
+        """Convert the entire APDU to a dict."""
+        return {'cmd': self.cmd_dict, 'rsp': self.rsp_dict}
+
+    def to_json(self) -> str:
+        """Convert the entire APDU to JSON."""
+        d = self.to_dict()
+        return json.dumps(d)
+
+    def _determine_file(self, lchan) -> CardFile:
+        """Helper function for read/update commands that might use SFI instead of selected file.
+        Expects that the self.cmd_dict has already been populated with the 'file' member."""
+        if self.cmd_dict['file'] == 'currently_selected_ef':
+            self.file = lchan.selected_file
+        elif self.cmd_dict['file'] == 'sfi':
+            cwd = lchan.get_cwd()
+            self.file = cwd.lookup_file_by_sfid(self.cmd_dict['sfi'])
+
+
+class ApduCommandSet:
+    """A set of card instructions, typically specified within one spec."""
+
+    def __init__(self, name: str, cmds: List[ApduCommand] =[]):
+        self.name = name
+        self.cmds = {c._ins: c for c in cmds}
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __getitem__(self, idx) -> ApduCommand:
+        return self.cmds[idx]
+
+    def __add__(self, other) -> 'ApduCommandSet':
+        if isinstance(other, ApduCommand):
+            if other.ins in self.cmds:
+                raise ValueError('%s: INS 0x%02x already defined: %s' %
+                                 (self, other.ins, self.cmds[other.ins]))
+            self.cmds[other.ins] = other
+        elif isinstance(other, ApduCommandSet):
+            for c in other.cmds.keys():
+                self.cmds[c] = other.cmds[c]
+        else:
+            raise ValueError(
+                '%s: Unsupported type to add operator: %s' % (self, other))
+        return self
+
+    def lookup(self, ins, cla=None) -> Optional[ApduCommand]:
+        """look-up the command within the CommandSet."""
+        ins = int(ins)
+        if not ins in self.cmds:
+            return None
+        cmd = self.cmds[ins]
+        if cla and not cmd.match_cla(cla):
+            return None
+        return cmd
+
+    def parse_cmd_apdu(self, apdu: Apdu) -> ApduCommand:
+        """Parse a Command-APDU. Returns an instance of an ApduCommand derived class."""
+        # first look-up which of our member classes match CLA + INS
+        a_cls = self.lookup(apdu.ins, apdu.cla)
+        if not a_cls:
+            raise ValueError('Unknown CLA=%02X INS=%02X' % (apdu.cla, apdu.ins))
+        # then create an instance of that class and return it
+        return a_cls.from_apdu(apdu)
+
+    def parse_cmd_bytes(self, buf:bytes) -> ApduCommand:
+        """Parse from a buffer (simtrace style). Returns an instance of an ApduCommand derived class."""
+        # first look-up which of our member classes match CLA + INS
+        cla = buf[0]
+        ins = buf[1]
+        a_cls = self.lookup(ins, cla)
+        if not a_cls:
+            raise ValueError('Unknown CLA=%02X INS=%02X' % (cla, ins))
+        # then create an instance of that class and return it
+        return a_cls.from_bytes(buf)
+
+
+
+class ApduHandler(abc.ABC):
+    @abc.abstractmethod
+    def input(self, cmd: bytes, rsp: bytes):
+        pass
+
+
+class TpduFilter(ApduHandler):
+    """The TpduFilter removes the T=0 specific GET_RESPONSE from the TPDU stream and
+       calls the ApduHandler only with the actual APDU command and response parts."""
+    def __init__(self, apdu_handler: ApduHandler):
+        self.apdu_handler = apdu_handler
+        self.state = 'INIT'
+        self.last_cmd = None
+
+    def input_tpdu(self, tpdu:Tpdu):
+        # handle SW=61xx / 6Cxx
+        if tpdu.sw[0] == 0x61 or tpdu.sw[0] == 0x6C:
+            self.state = 'WAIT_GET_RESPONSE'
+            # handle successive 61/6c responses by stupid phone/modem OS
+            if tpdu.ins != 0xC0:
+                self.last_cmd = tpdu.cmd
+            return None
+        else:
+            if self.last_cmd:
+                icmd = self.last_cmd
+                self.last_cmd = None
+            else:
+                icmd = tpdu.cmd
+            apdu = Apdu(icmd, tpdu.rsp)
+            if self.apdu_handler:
+                return self.apdu_handler.input(apdu)
+            else:
+                return Apdu(icmd, tpdu.rsp)
+
+    def input(self, cmd: bytes, rsp: bytes):
+        if isinstance(cmd, str):
+            cmd = bytes.fromhex(cmd)
+        if isinstance(rsp, str):
+            rsp = bytes.fromhex(rsp)
+        tpdu = Tpdu(cmd, rsp)
+        return self.input_tpdu(tpdu)
+
+class ApduDecoder(ApduHandler):
+    def __init__(self, cmd_set: ApduCommandSet):
+        self.cmd_set = cmd_set
+
+    def input(self, apdu: Apdu):
+        return self.cmd_set.parse_cmd_apdu(apdu)
+
+
+class CardReset:
+    pass

pySim/apdu/global_platform.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+"""APDU definition/decoder of GlobalPLatform Card Spec (currently 2.1.1)
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+class GpDelete(ApduCommand, n='DELETE', ins=0xE4, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpStoreData(ApduCommand, n='STORE DATA', ins=0xE2, cla=['8X', 'CX', 'EX']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        if p1 & 0x01:
+            return 4
+        else:
+            return 3
+
+class GpGetDataCA(ApduCommand, n='GET DATA', ins=0xCA, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpGetDataCB(ApduCommand, n='GET DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpGetStatus(ApduCommand, n='GET STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpInstall(ApduCommand, n='INSTALL', ins=0xE6, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpLoad(ApduCommand, n='LOAD', ins=0xE8, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpPutKey(ApduCommand, n='PUT KEY', ins=0xD8, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpSetStatus(ApduCommand, n='SET STATUS', ins=0xF0, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+ApduCommands = ApduCommandSet('GlobalPlatform v2.3.1', cmds=[GpDelete, GpStoreData,
+                              GpGetDataCA, GpGetDataCB, GpGetStatus, GpInstall,
+                              GpLoad, GpPutKey, GpSetStatus])

pySim/apdu/ts_102_221.py

@@ -0,0 +1,525 @@
+# coding=utf-8
+"""APDU definitions/decoders of ETSI TS 102 221, the core UICC spec.
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import logging
+from pySim.construct import *
+from pySim.filesystem import *
+from pySim.apdu import ApduCommand, ApduCommandSet
+from typing import Optional, Dict, Tuple
+
+logger = logging.getLogger(__name__)
+
+# TS 102 221 Section 11.1.1
+class UiccSelect(ApduCommand, n='SELECT', ins=0xA4, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p1 = Enum(Byte, df_ef_or_mf_by_file_id=0, child_df_of_current_df=1, parent_df_of_current_df=3,
+                         df_name=4, path_from_mf=8, path_from_current_df=9)
+    _construct_p2 = BitStruct(Flag,
+                              'app_session_control'/Enum(BitsInteger(2), activation_reset=0, termination=2),
+                              'return'/Enum(BitsInteger(3), fcp=1, no_data=3),
+                              'aid_control'/Enum(BitsInteger(2), first_or_only=0, last=1, next=2, previous=3))
+
+    @staticmethod
+    def _find_aid_substr(selectables, aid) -> Optional[CardADF]:
+        # full-length match
+        if aid in selectables:
+            return selectables[aid]
+        # sub-string match
+        for s in selectables.keys():
+            if aid[:len(s)] == s:
+                return selectables[s]
+        return None
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        mode = self.cmd_dict['p1']
+        if mode in ['path_from_mf', 'path_from_current_df']:
+            # rewind to MF, if needed
+            if mode == 'path_from_mf':
+                lchan.selected_file = lchan.rs.mf
+            path = [self.cmd_data[i:i+2] for i in range(0, len(self.cmd_data), 2)]
+            for file in path:
+                file_hex = b2h(file)
+                if file_hex == '7fff': # current application
+                    if not lchan.selected_adf:
+                        sels = lchan.rs.mf.get_app_selectables(['ANAMES'])
+                        # HACK: Assume USIM
+                        logger.warning('SELECT relative to current ADF, but no ADF selected. Assuming ADF.USIM')
+                        lchan.selected_adf = sels['ADF.USIM']
+                    lchan.selected_file = lchan.selected_adf
+                    #print("\tSELECT CUR_ADF %s" % lchan.selected_file)
+                    # iterate to next element in path
+                    continue
+                else:
+                    sels = lchan.selected_file.get_selectables(['FIDS','MF','PARENT','SELF'])
+                    if file_hex in sels:
+                        if self.successful:
+                            #print("\tSELECT %s" % sels[file_hex])
+                            lchan.selected_file = sels[file_hex]
+                        else:
+                            #print("\tSELECT %s FAILED" % sels[file_hex])
+                            pass
+                        # iterate to next element in path
+                        continue
+                logger.warning('SELECT UNKNOWN FID %s (%s)' % (file_hex, '/'.join([b2h(x) for x in path])))
+        elif mode == 'df_ef_or_mf_by_file_id':
+            if len(self.cmd_data) != 2:
+                raise ValueError('Expecting a 2-byte FID')
+            sels = lchan.selected_file.get_selectables(['FIDS','MF','PARENT','SELF'])
+            file_hex = b2h(self.cmd_data)
+            if file_hex in sels:
+                if self.successful:
+                    #print("\tSELECT %s" % sels[file_hex])
+                    lchan.selected_file = sels[file_hex]
+                else:
+                    #print("\tSELECT %s FAILED" % sels[file_hex])
+                    pass
+            else:
+                logger.warning('SELECT UNKNOWN FID %s' % (file_hex))
+        elif mode == 'df_name':
+            # Select by AID (can be sub-string!)
+            aid = self.cmd_dict['body']
+            sels = lchan.rs.mf.get_app_selectables(['AIDS'])
+            adf = self._find_aid_substr(sels, aid)
+            if adf:
+                lchan.selected_adf = adf
+                lchan.selected_file = lchan.selected_adf
+                #print("\tSELECT AID %s" % adf)
+            else:
+                logger.warning('SELECT UNKNOWN AID %s' % aid)
+                pass
+        else:
+            raise ValueError('Select Mode %s not implemented' % mode)
+        # decode the SELECT response
+        if self.successful:
+            self.file = lchan.selected_file
+            if 'body' in self.rsp_dict:
+                # not every SELECT is asking for the FCP in response...
+                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+        return None
+
+
+
+# TS 102 221 Section 11.1.2
+class UiccStatus(ApduCommand, n='STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 2
+    _construct_p1 = Enum(Byte, no_indication=0, current_app_is_initialized=1, terminal_will_terminate_current_app=2)
+    _construct_p2 = Enum(Byte, response_like_select=0, response_df_name_tlv=1, response_no_data=0x0c)
+
+    def process_on_lchan(self, lchan):
+        if self.cmd_dict['p2'] == 'response_like_select':
+            return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+
+def _decode_binary_p1p2(p1, p2) -> Dict:
+    ret = {}
+    if p1 & 0x80:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p1 & 0x1f
+        ret['offset'] = p2
+    else:
+        ret['file'] = 'currently_selected_ef'
+        ret['offset'] = ((p1 & 0x7f) << 8) & p2
+    return ret
+
+# TS 102 221 Section 11.1.3
+class ReadBinary(ApduCommand, n='READ BINARY', ins=0xB0, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short reads
+        if self.cmd_dict['offset'] != 0 or self.lr < self.file.size[0]:
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 102 221 Section 11.1.4
+class UpdateBinary(ApduCommand, n='UPDATE BINARY', ins=0xD6, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short writes
+        if self.cmd_dict['offset'] != 0 or self.lc < self.file.size[0]:
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+def _decode_record_p1p2(p1, p2):
+    ret = {}
+    ret['record_number'] = p1
+    if p2 >> 3 == 0:
+        ret['file'] = 'currently_selected_ef'
+    else:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p2 >> 3
+    mode = p2 & 0x7
+    if mode == 2:
+        ret['mode'] = 'next_record'
+    elif mode == 3:
+        ret['mode'] = 'previous_record'
+    elif mode == 8:
+        ret['mode'] = 'absolute_current'
+    return ret
+
+# TS 102 221 Section 11.1.5
+class ReadRecord(ApduCommand, n='READ RECORD', ins=0xB2, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 102 221 Section 11.1.6
+class UpdateRecord(ApduCommand, n='UPDATE RECORD', ins=0xDC, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+# TS 102 221 Section 11.1.7
+class SearchRecord(ApduCommand, n='SEARCH RECORD', ins=0xA2, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_rsp = GreedyRange(Int8ub)
+
+    def _decode_p1p2(self):
+        ret = {}
+        sfi = self.p2 >> 3
+        if sfi == 0:
+            ret['file'] = 'currently_selected_ef'
+        else:
+            ret['file'] = 'sfi'
+            ret['sfi'] = sfi
+        mode = self.p2 & 0x7
+        if mode in [0x4, 0x5]:
+            if mode == 0x4:
+                ret['mode'] = 'forward_search'
+            else:
+                ret['mode'] = 'backward_search'
+            ret['record_number'] = self.p1
+            self.col_id = '%02u' % ret['record_number']
+        elif mode == 6:
+            ret['mode'] = 'enhanced_search'
+            # TODO: further decode
+        elif mode == 7:
+            ret['mode'] = 'proprietary_search'
+        return ret
+
+    def _decode_cmd(self):
+        ret = self._decode_p1p2()
+        if self.cmd_data:
+            if ret['mode'] == 'enhanced_search':
+                ret['search_indication'] = b2h(self.cmd_data[:2])
+                ret['search_string'] = b2h(self.cmd_data[2:])
+            else:
+                ret['search_string'] = b2h(self.cmd_data)
+        return ret
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        return self.to_dict()
+
+# TS 102 221 Section 11.1.8
+class Increase(ApduCommand, n='INCREASE', ins=0x32, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+PinConstructP2 = BitStruct('scope'/Enum(Flag, global_mf=0, specific_df_adf=1),
+                           BitsInteger(2), 'reference_data_nr'/BitsInteger(5))
+# TS 102 221 Section 11.1.9
+class VerifyPin(ApduCommand, n='VERIFY PIN', ins=0x20, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    @staticmethod
+    def _pin_process(apdu):
+        processed = {
+            'scope': apdu.cmd_dict['p2']['scope'],
+            'referenced_data_nr': apdu.cmd_dict['p2']['reference_data_nr'],
+            }
+        if apdu.lc == 0:
+            # this is just a question on the counters remaining
+            processed['mode'] = 'check_remaining_attempts'
+        else:
+            processed['pin'] = b2h(apdu.cmd_data)
+        if apdu.sw[0] == 0x63:
+            processed['remaining_attempts'] = apdu.sw[1] & 0xf
+        return processed
+
+    @staticmethod
+    def _pin_is_success(sw):
+        if sw[0] == 0x63:
+            return True
+        else:
+            return False
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.10
+class ChangePin(ApduCommand, n='CHANGE PIN', ins=0x24, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.11
+class DisablePin(ApduCommand, n='DISABLE PIN', ins=0x26, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.12
+class EnablePin(ApduCommand, n='ENABLE PIN', ins=0x28, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.13
+class UnblockPin(ApduCommand, n='UNBLOCK PIN', ins=0x2C, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.14
+class DeactivateFile(ApduCommand, n='DEACTIVATE FILE', ins=0x04, cla=['0X', '4X', '6X']):
+    _apdu_case = 1
+    _construct_p1 = BitStruct(BitsInteger(4),
+                              'select_mode'/Enum(BitsInteger(4), ef_by_file_id=0,
+                                                 path_from_mf=8, path_from_current_df=9))
+
+# TS 102 221 Section 11.1.15
+class ActivateFile(ApduCommand, n='ACTIVATE FILE', ins=0x44, cla=['0X', '4X', '6X']):
+    _apdu_case = 1
+    _construct_p1 = DeactivateFile._construct_p1
+
+# TS 102 221 Section 11.1.16
+auth_p2_construct = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(2),
+                              'reference_data_nr'/BitsInteger(5))
+class Authenticate88(ApduCommand, n='AUTHENTICATE', ins=0x88, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = auth_p2_construct
+
+# TS 102 221 Section 11.1.16
+class Authenticate89(ApduCommand, n='AUTHENTICATE', ins=0x89, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = auth_p2_construct
+
+# TS 102 221 Section 11.1.17
+class ManageChannel(ApduCommand, n='MANAGE CHANNEL', ins=0x70, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    _construct_p1 = Enum(Flag, open_channel=0, close_channel=1)
+    _construct_p2 = Struct('logical_channel_number'/Int8ub)
+    _construct_rsp = Struct('logical_channel_number'/Int8ub)
+
+    def process_global(self, rs):
+        if not self.successful:
+            return
+        mode = self.cmd_dict['p1']
+        if mode == 'open_channel':
+            created_channel_nr = self.cmd_dict['p2']['logical_channel_number']
+            if created_channel_nr == 0:
+                # auto-assignment by UICC
+                # pylint: disable=unsubscriptable-object
+                created_channel_nr = self.rsp_data[0]
+            manage_channel = rs.get_lchan_by_cla(self.cla)
+            manage_channel.add_lchan(created_channel_nr)
+            self.col_id = '%02u' % created_channel_nr
+        elif mode == 'close_channel':
+            closed_channel_nr = self.cmd_dict['p2']
+            rs.del_lchan(closed_channel_nr)
+            self.col_id = '%02u' % closed_channel_nr
+        else:
+            raise ValueError('Unsupported MANAGE CHANNEL P1=%02X' % self.p1)
+
+# TS 102 221 Section 11.1.18
+class GetChallenge(ApduCommand, n='GET CHALLENGE', ins=0x84, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+
+# TS 102 221 Section 11.1.19
+class TerminalCapability(ApduCommand, n='TERMINAL CAPABILITY', ins=0xAA, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.1.20
+class ManageSecureChannel(ApduCommand, n='MANAGE SECURE CHANNEL', ins=0x73, cla=['0X', '4X', '6X']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        p2 = hdr[3]
+        if p1 & 0x7 == 0:   # retrieve UICC Endpoints
+            return 2
+        elif p1 & 0xf in [1,2,3]:   # establish sa, start secure channel SA
+            p2_cmd = p2 >> 5
+            if p2_cmd in [0,2,4]:   # command data
+                return 3
+            elif p2_cmd in [1,3,5]: # response data
+                return 2
+        elif p1 & 0xf == 4:         # terminate secure channel SA
+            return 3
+        raise ValueError('%s: Unable to detect APDU case for %s' % (cls.__name__, b2h(hdr)))
+
+# TS 102 221 Section 11.1.21
+class TransactData(ApduCommand, n='TRANSACT DATA', ins=0x75, cla=['0X', '4X', '6X']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        if p1 & 0x04:
+            return 3
+        else:
+            return 2
+
+# TS 102 221 Section 11.1.22
+class SuspendUicc(ApduCommand, n='SUSPEND UICC', ins=0x76, cla=['80']):
+    _apdu_case = 4
+    _construct_p1 = BitStruct('rfu'/BitsInteger(7), 'mode'/Enum(Flag, suspend=0, resume=1))
+
+# TS 102 221 Section 11.1.23
+class GetIdentity(ApduCommand, n='GET IDENTITY', ins=0x78, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1), BitsInteger(7))
+
+# TS 102 221 Section 11.1.24
+class ExchangeCapabilities(ApduCommand, n='EXCHANGE CAPABILITIES', ins=0x7A, cla=['80']):
+    _apdu_case = 4
+
+# TS 102 221 Section 11.2.1
+class TerminalProfile(ApduCommand, n='TERMINAL PROFILE', ins=0x10, cla=['80']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.2.2 / TS 102 223
+class Envelope(ApduCommand, n='ENVELOPE', ins=0xC2, cla=['80']):
+    _apdu_case = 4
+
+# TS 102 221 Section 11.2.3 / TS 102 223
+class Fetch(ApduCommand, n='FETCH', ins=0x12, cla=['80']):
+    _apdu_case = 2
+
+# TS 102 221 Section 11.2.3 / TS 102 223
+class TerminalResponse(ApduCommand, n='TERMINAL RESPONSE', ins=0x14, cla=['80']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.3.1
+class RetrieveData(ApduCommand, n='RETRIEVE DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+    @staticmethod
+    def _tlv_decode_cmd(self : ApduCommand) -> Dict:
+        c = {}
+        if self.p2 & 0xc0 == 0x80:
+            c['mode'] = 'first_block'
+            sfi = self.p2 & 0x1f
+            if sfi == 0:
+                c['file'] = 'currently_selected_ef'
+            else:
+                c['file'] = 'sfi'
+                c['sfi'] = sfi
+            c['tag'] = i2h([self.cmd_data[0]])
+        elif self.p2 & 0xdf == 0x00:
+            c['mode'] = 'next_block'
+        elif self.p2 & 0xdf == 0x40:
+            c['mode'] = 'retransmit_previous_block'
+        else:
+            logger.warning('%s: invalid P2=%02x' % (self, self.p2))
+        return c
+
+    def _decode_cmd(self):
+        return RetrieveData._tlv_decode_cmd(self)
+
+    def _decode_rsp(self):
+        # TODO: parse tag/len/val?
+        return b2h(self.rsp_data)
+
+
+# TS 102 221 Section 11.3.2
+class SetData(ApduCommand, n='SET DATA', ins=0xDB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+    def _decode_cmd(self):
+        c = RetrieveData._tlv_decode_cmd(self)
+        if c['mode'] == 'first_block':
+            if len(self.cmd_data) == 0:
+                c['delete'] = True
+        # TODO: parse tag/len/val?
+        c['data'] = b2h(self.cmd_data)
+        return c
+
+
+# TS 102 221 Section 12.1.1
+class GetResponse(ApduCommand, n='GET RESPONSE', ins=0xC0, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+
+ApduCommands = ApduCommandSet('TS 102 221', cmds=[UiccSelect, UiccStatus, ReadBinary, UpdateBinary, ReadRecord,
+                              UpdateRecord, SearchRecord, Increase, VerifyPin, ChangePin, DisablePin,
+                              EnablePin, UnblockPin, DeactivateFile, ActivateFile, Authenticate88,
+                              Authenticate89, ManageChannel, GetChallenge, TerminalCapability,
+                              ManageSecureChannel, TransactData, SuspendUicc, GetIdentity,
+                              ExchangeCapabilities, TerminalProfile, Envelope, Fetch, TerminalResponse,
+                              RetrieveData, SetData, GetResponse])

pySim/apdu/ts_31_102.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+APDU commands of 3GPP TS 31.102 V16.6.0
+"""
+from typing import Dict
+
+from construct import *
+from construct import Optional as COptional
+from pySim.filesystem import *
+from pySim.construct import *
+from pySim.ts_31_102 import SUCI_TlvDataObject
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+# Copyright (C) 2022 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Mapping between USIM Service Number and its description
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+# TS 31.102 Section 7.1
+class UsimAuthenticateEven(ApduCommand, n='AUTHENTICATE', ins=0x88, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(4),
+                              'authentication_context'/Enum(BitsInteger(3), gsm=0, umts=1,
+                                                            vgcs_vbs=2, gba=4))
+    _cs_cmd_gsm_3g =  Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
+                         '_autn_len'/COptional(Int8ub), 'autn'/If(this._autn_len, HexAdapter(Bytes(this._autn_len))))
+    _cs_cmd_vgcs = Struct('_vsid_len'/Int8ub, 'vservice_id'/HexAdapter(Bytes(this._vsid_len)),
+                          '_vkid_len'/Int8ub, 'vk_id'/HexAdapter(Bytes(this._vkid_len)),
+                          '_vstk_rand_len'/Int8ub, 'vstk_rand'/HexAdapter(Bytes(this._vstk_rand_len)))
+    _cmd_gba_bs = Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
+                         '_autn_len'/Int8ub, 'autn'/HexAdapter(Bytes(this._autn_len)))
+    _cmd_gba_naf = Struct('_naf_id_len'/Int8ub, 'naf_id'/HexAdapter(Bytes(this._naf_id_len)),
+                          '_impi_len'/Int8ub, 'impi'/HexAdapter(Bytes(this._impi_len)))
+    _cs_cmd_gba = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDD: 'bootstrap'/_cmd_gba_bs,
+                                                                 0xDE: 'naf_derivation'/_cmd_gba_naf }))
+    _cs_rsp_gsm = Struct('_len_sres'/Int8ub, 'sres'/HexAdapter(Bytes(this._len_sres)),
+                         '_len_kc'/Int8ub, 'kc'/HexAdapter(Bytes(this._len_kc)))
+    _rsp_3g_ok = Struct('_len_res'/Int8ub, 'res'/HexAdapter(Bytes(this._len_res)),
+                        '_len_ck'/Int8ub, 'ck'/HexAdapter(Bytes(this._len_ck)),
+                        '_len_ik'/Int8ub, 'ik'/HexAdapter(Bytes(this._len_ik)),
+                        '_len_kc'/COptional(Int8ub), 'kc'/If(this._len_kc, HexAdapter(Bytes(this._len_kc))))
+    _rsp_3g_sync = Struct('_len_auts'/Int8ub, 'auts'/HexAdapter(Bytes(this._len_auts)))
+    _cs_rsp_3g = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDB: 'success'/_rsp_3g_ok,
+                                                                0xDC: 'sync_fail'/_rsp_3g_sync}))
+    _cs_rsp_vgcs = Struct(Const(b'\xDB'), '_vstk_len'/Int8ub, 'vstk'/HexAdapter(Bytes(this._vstk_len)))
+    _cs_rsp_gba_naf = Struct(Const(b'\xDB'), '_ks_ext_naf_len'/Int8ub, 'ks_ext_naf'/HexAdapter(Bytes(this._ks_ext_naf_len)))
+    def _decode_cmd(self) -> Dict:
+        r = {}
+        r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
+        r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
+        auth_ctx = r['p2']['authentication_context']
+        if auth_ctx in ['gsm', 'umts']:
+            r['body'] = parse_construct(self._cs_cmd_gsm_3g, self.cmd_data)
+        elif auth_ctx == 'vgcs_vbs':
+            r['body'] = parse_construct(self._cs_cmd_vgcs, self.cmd_data)
+        elif auth_ctx == 'gba':
+            r['body'] = parse_construct(self._cs_cmd_gba, self.cmd_data)
+        else:
+            raise ValueError('Unsupported authentication_context: %s' % auth_ctx)
+        return r
+
+    def _decode_rsp(self) -> Dict:
+        r = {}
+        auth_ctx = self.cmd_dict['p2']['authentication_context']
+        if auth_ctx == 'gsm':
+            r['body'] = parse_construct(self._cs_rsp_gsm, self.rsp_data)
+        elif auth_ctx == 'umts':
+            r['body'] = parse_construct(self._cs_rsp_3g, self.rsp_data)
+        elif auth_ctx == 'vgcs_vbs':
+            r['body'] = parse_construct(self._cs_rsp_vgcs, self.rsp_data)
+        elif auth_ctx == 'gba':
+            if self.cmd_dict['body']['tag'] == 0xDD:
+                r['body'] = parse_construct(self._cs_rsp_3g, self.rsp_data)
+            else:
+                r['body'] = parse_construct(self._cs_rsp_gba_naf, self.rsp_data)
+        else:
+            raise ValueError('Unsupported authentication_context: %s' % auth_ctx)
+        return r
+
+class UsimAuthenticateOdd(ApduCommand, n='AUTHENTICATE', ins=0x89, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(4),
+                              'authentication_context'/Enum(BitsInteger(3), mbms=5, local_key=6))
+# TS 31.102 Section 7.5
+class UsimGetIdentity(ApduCommand, n='GET IDENTITY', ins=0x78, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              'identity_context'/Enum(BitsInteger(7), suci=1))
+    _tlv_rsp = SUCI_TlvDataObject
+
+ApduCommands = ApduCommandSet('TS 31.102', cmds=[UsimAuthenticateEven, UsimAuthenticateOdd,
+                                                 UsimGetIdentity])

pySim/apdu/ts_51_011.py

@@ -0,0 +1,337 @@
+# coding=utf-8
+"""APDU definitions/decoders of 3GPP TS 51.011, the classic SIM spec.
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import logging
+from pySim.construct import *
+from pySim.filesystem import *
+from pySim.apdu import ApduCommand, ApduCommandSet
+from typing import Optional, Dict, Tuple
+
+logger = logging.getLogger(__name__)
+
+# TS 51.011 Section 9.2.1
+class SimSelect(ApduCommand, n='SELECT', ins=0xA4, cla=['A0']):
+    _apdu_case = 4
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        path = [self.cmd_data[i:i+2] for i in range(0, len(self.cmd_data), 2)]
+        for file in path:
+            file_hex = b2h(file)
+            sels = lchan.selected_file.get_selectables(['FIDS'])
+            if file_hex in sels:
+                if self.successful:
+                    #print("\tSELECT %s" % sels[file_hex])
+                    lchan.selected_file = sels[file_hex]
+                else:
+                    #print("\tSELECT %s FAILED" % sels[file_hex])
+                    pass
+                continue
+        logger.warning('SELECT UNKNOWN FID %s (%s)' % (file_hex, '/'.join([b2h(x) for x in path])))
+        if len(self.cmd_data) != 2:
+            raise ValueError('Expecting a 2-byte FID')
+
+        # decode the SELECT response
+        if self.successful:
+            self.file = lchan.selected_file
+            if 'body' in self.rsp_dict:
+                # not every SELECT is asking for the FCP in response...
+                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+        return None
+
+
+# TS 51.011 Section 9.2.2
+class SimStatus(ApduCommand, n='STATUS', ins=0xF2, cla=['A0']):
+    _apdu_case = 2
+
+    def process_on_lchan(self, lchan):
+        if self.successful:
+            if 'body' in self.rsp_dict:
+                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+
+def _decode_binary_p1p2(p1, p2) -> Dict:
+    ret = {}
+    if p1 & 0x80:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p1 & 0x1f
+        ret['offset'] = p2
+    else:
+        ret['file'] = 'currently_selected_ef'
+        ret['offset'] = ((p1 & 0x7f) << 8) & p2
+    return ret
+
+# TS 51.011 Section 9.2.3 / 31.101
+class ReadBinary(ApduCommand, n='READ BINARY', ins=0xB0, cla=['A0']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short reads
+        if self.cmd_dict['offset'] != 0 or self.lr < self.file.size[0]:
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 51.011 Section 9.2.4 / 31.101
+class UpdateBinary(ApduCommand, n='UPDATE BINARY', ins=0xD6, cla=['A0']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short writes
+        if self.cmd_dict['offset'] != 0 or self.lc < self.file.size[0]:
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+def _decode_record_p1p2(p1, p2):
+    ret = {}
+    ret['record_number'] = p1
+    if p2 >> 3 == 0:
+        ret['file'] = 'currently_selected_ef'
+    else:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p2 >> 3
+    mode = p2 & 0x7
+    if mode == 2:
+        ret['mode'] = 'next_record'
+    elif mode == 3:
+        ret['mode'] = 'previous_record'
+    elif mode == 8:
+        ret['mode'] = 'absolute_current'
+    return ret
+
+# TS 51.011 Section 9.2.5
+class ReadRecord(ApduCommand, n='READ RECORD', ins=0xB2, cla=['A0']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 51.011 Section 9.2.6
+class UpdateRecord(ApduCommand, n='UPDATE RECORD', ins=0xDC, cla=['A0']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+# TS 51.011 Section 9.2.7
+class Seek(ApduCommand, n='SEEK', ins=0xA2, cla=['A0']):
+    _apdu_case = 4
+    _construct_rsp = GreedyRange(Int8ub)
+
+    def _decode_p1p2(self):
+        ret = {}
+        sfi = self.p2 >> 3
+        if sfi == 0:
+            ret['file'] = 'currently_selected_ef'
+        else:
+            ret['file'] = 'sfi'
+            ret['sfi'] = sfi
+        mode = self.p2 & 0x7
+        if mode in [0x4, 0x5]:
+            if mode == 0x4:
+                ret['mode'] = 'forward_search'
+            else:
+                ret['mode'] = 'backward_search'
+            ret['record_number'] = self.p1
+            self.col_id = '%02u' % ret['record_number']
+        elif mode == 6:
+            ret['mode'] = 'enhanced_search'
+            # TODO: further decode
+        elif mode == 7:
+            ret['mode'] = 'proprietary_search'
+        return ret
+
+    def _decode_cmd(self):
+        ret = self._decode_p1p2()
+        if self.cmd_data:
+            if ret['mode'] == 'enhanced_search':
+                ret['search_indication'] = b2h(self.cmd_data[:2])
+                ret['search_string'] = b2h(self.cmd_data[2:])
+            else:
+                ret['search_string'] = b2h(self.cmd_data)
+        return ret
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        return self.to_dict()
+
+# TS 51.011 Section 9.2.8
+class Increase(ApduCommand, n='INCREASE', ins=0x32, cla=['A0']):
+    _apdu_case = 4
+
+PinConstructP2 = BitStruct('scope'/Enum(Flag, global_mf=0, specific_df_adf=1),
+                           BitsInteger(2), 'reference_data_nr'/BitsInteger(5))
+
+# TS 51.011 Section 9.2.9
+class VerifyChv(ApduCommand, n='VERIFY CHV', ins=0x20, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    @staticmethod
+    def _pin_process(apdu):
+        processed = {
+            'scope': apdu.cmd_dict['p2']['scope'],
+            'referenced_data_nr': apdu.cmd_dict['p2']['reference_data_nr'],
+            }
+        if apdu.lc == 0:
+            # this is just a question on the counters remaining
+            processed['mode'] = 'check_remaining_attempts'
+        else:
+            processed['pin'] = b2h(apdu.cmd_data)
+        if apdu.sw[0] == 0x63:
+            processed['remaining_attempts'] = apdu.sw[1] & 0xf
+        return processed
+
+    @staticmethod
+    def _pin_is_success(sw):
+        if sw[0] == 0x63:
+            return True
+        else:
+            return False
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.10
+class ChangeChv(ApduCommand, n='CHANGE CHV', ins=0x24, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.11
+class DisableChv(ApduCommand, n='DISABLE CHV', ins=0x26, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.12
+class EnableChv(ApduCommand, n='ENABLE CHV', ins=0x28, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.13
+class UnblockChv(ApduCommand, n='UNBLOCK CHV', ins=0x2C, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.14
+class Invalidate(ApduCommand, n='INVALIDATE', ins=0x04, cla=['A0']):
+    _apdu_case = 1
+    _construct_p1 = BitStruct(BitsInteger(4),
+                              'select_mode'/Enum(BitsInteger(4), ef_by_file_id=0,
+                                                 path_from_mf=8, path_from_current_df=9))
+
+# TS 51.011 Section 9.2.15
+class Rehabilitate(ApduCommand, n='REHABILITATE', ins=0x44, cla=['A0']):
+    _apdu_case = 1
+    _construct_p1 = Invalidate._construct_p1
+
+# TS 51.011 Section 9.2.16
+class RunGsmAlgorithm(ApduCommand, n='RUN GSM ALGORITHM', ins=0x88, cla=['A0']):
+    _apdu_case = 4
+    _construct = Struct('rand'/HexAdapter(Bytes(16)))
+    _construct_rsp = Struct('sres'/HexAdapter(Bytes(4)), 'kc'/HexAdapter(Bytes(8)))
+
+# TS 51.011 Section 9.2.17
+class Sleep(ApduCommand, n='SLEEP', ins=0xFA, cla=['A0']):
+    _apdu_case = 2
+
+# TS 51.011 Section 9.2.18
+class GetResponse(ApduCommand, n='GET RESPONSE', ins=0xC0, cla=['A0']):
+    _apdu_case = 2
+
+# TS 51.011 Section 9.2.19
+class TerminalProfile(ApduCommand, n='TERMINAL PROFILE', ins=0x10, cla=['A0']):
+    _apdu_case = 3
+
+# TS 51.011 Section 9.2.20
+class Envelope(ApduCommand, n='ENVELOPE', ins=0xC2, cla=['A0']):
+    _apdu_case = 4
+
+# TS 51.011 Section 9.2.21
+class Fetch(ApduCommand, n='FETCH', ins=0x12, cla=['A0']):
+    _apdu_case = 2
+
+# TS 51.011 Section 9.2.22
+class TerminalResponse(ApduCommand, n='TERMINAL RESPONSE', ins=0x14, cla=['A0']):
+    _apdu_case = 3
+
+
+ApduCommands = ApduCommandSet('TS 51.011', cmds=[SimSelect, SimStatus, ReadBinary, UpdateBinary, ReadRecord,
+                              UpdateRecord, Seek, Increase, VerifyChv, ChangeChv, DisableChv,
+                              EnableChv, UnblockChv, Invalidate, Rehabilitate, RunGsmAlgorithm,
+                              Sleep, GetResponse, TerminalProfile, Envelope, Fetch, TerminalResponse])

pySim/apdu_source/__init__.py

@@ -0,0 +1,35 @@
+import abc
+import logging
+from typing import Union
+from pySim.apdu import Apdu, Tpdu, CardReset, TpduFilter
+
+PacketType = Union[Apdu, Tpdu, CardReset]
+
+logger = logging.getLogger(__name__)
+
+class ApduSource(abc.ABC):
+    def __init__(self):
+        self.apdu_filter = TpduFilter(None)
+
+    @abc.abstractmethod
+    def read_packet(self) -> PacketType:
+        """Read one packet from the source."""
+        pass
+
+    def read(self) -> Union[Apdu, CardReset]:
+        """Main function to call by the user: Blocking read, returns Apdu or CardReset."""
+        apdu = None
+        # loop until we actually have an APDU to return
+        while not apdu:
+            r = self.read_packet()
+            if not r:
+                continue
+            if isinstance(r, Tpdu):
+                apdu = self.apdu_filter.input_tpdu(r)
+            elif isinstance(r, Apdu):
+                apdu = r
+            elif isinstance(r, CardReset):
+                apdu = r
+            else:
+                ValueError('Unknown read_packet() return %s' % r)
+        return apdu

pySim/apdu_source/gsmtap.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from pySim.gsmtap import GsmtapMessage, GsmtapSource
+from . import ApduSource, PacketType, CardReset
+
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+ApduCommands = UiccApduCommands + UsimApduCommands + GpApduCommands
+
+class GsmtapApduSource(ApduSource):
+    """ApduSource for handling GSMTAP-SIM messages received via UDP, such as
+    those generated by simtrace2-sniff.  Note that *if* you use IP loopback
+    and localhost addresses (which is the default), you will need to start
+    this source before starting simtrace2-sniff, as otherwise the latter will
+    claim the GSMTAP UDP port.
+    """
+    def __init__(self, bind_ip:str='127.0.0.1', bind_port:int=4729):
+        """Create a UDP socket for receiving GSMTAP-SIM messages.
+        Args:
+            bind_ip: IP address to which the socket should be bound (default: 127.0.0.1)
+            bind_port: UDP port number to which the socket should be bound (default: 4729)
+        """
+        super().__init__()
+        self.gsmtap = GsmtapSource(bind_ip, bind_port)
+
+    def read_packet(self) -> PacketType:
+        gsmtap_msg, addr = self.gsmtap.read_packet()
+        if gsmtap_msg['type'] != 'sim':
+            raise ValueError('Unsupported GSMTAP type %s' % gsmtap_msg['type'])
+        sub_type = gsmtap_msg['sub_type']
+        if sub_type == 'apdu':
+            return ApduCommands.parse_cmd_bytes(gsmtap_msg['body'])
+        elif sub_type == 'atr':
+            # card has been reset
+            return CardReset()
+        elif sub_type in ['pps_req', 'pps_rsp']:
+            # simply ignore for now
+            pass
+        else:
+            raise ValueError('Unsupported GSMTAP-SIM sub-type %s' % sub_type)

pySim/apdu_source/pyshark_rspro.py

@@ -0,0 +1,159 @@
+# coding=utf-8
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import sys
+import logging
+from pprint import pprint as pp
+from typing import Tuple
+import pyshark
+
+from pySim.utils import h2b, b2h
+from pySim.apdu import Tpdu
+from . import ApduSource, PacketType, CardReset
+
+logger = logging.getLogger(__name__)
+
+class _PysharkRspro(ApduSource):
+    """APDU Source [provider] base class for reading RSPRO (osmo-remsim) via tshark."""
+
+    def __init__(self, pyshark_inst):
+        self.pyshark = pyshark_inst
+        self.bank_id = None
+        self.bank_slot = None
+        self.cmd_tpdu = None
+        super().__init__()
+
+    @staticmethod
+    def get_bank_slot(bank_slot) -> Tuple[int, int]:
+        """Convert a 'bankSlot_element' field into a tuple of bank_id, slot_nr"""
+        bank_id = bank_slot.get_field('bankId')
+        slot_nr = bank_slot.get_field('slotNr')
+        return int(bank_id), int(slot_nr)
+
+    @staticmethod
+    def get_client_slot(client_slot) -> Tuple[int, int]:
+        """Convert a 'clientSlot_element' field into a tuple of client_id, slot_nr"""
+        client_id = client_slot.get_field('clientId')
+        slot_nr = client_slot.get_field('slotNr')
+        return int(client_id), int(slot_nr)
+
+    @staticmethod
+    def get_pstatus(pstatus) -> Tuple[int, int, int]:
+        """Convert a 'slotPhysStatus_element' field into a tuple of vcc, reset, clk"""
+        vccPresent = int(pstatus.get_field('vccPresent'))
+        resetActive = int(pstatus.get_field('resetActive'))
+        clkActive = int(pstatus.get_field('clkActive'))
+        return vccPresent, resetActive, clkActive
+
+    def read_packet(self) -> PacketType:
+        p = self.pyshark.next()
+        return self._parse_packet(p)
+
+    def _set_or_verify_bank_slot(self, bsl: Tuple[int, int]):
+        """Keep track of the bank:slot to make sure we don't mix traces of multiple cards"""
+        if not self.bank_id:
+            self.bank_id = bsl[0]
+            self.bank_slot = bsl[1]
+        else:
+            if self.bank_id != bsl[0] or self.bank_slot != bsl[1]:
+                raise ValueError('Received data for unexpected B(%u:%u)' % (bsl[0], bsl[1]))
+
+    def _parse_packet(self, p) -> PacketType:
+        rspro_layer = p['rspro']
+        #print("Layer: %s" %  rspro_layer)
+        rspro_element = rspro_layer.get_field('RsproPDU_element')
+        #print("Element: %s" % rspro_element)
+        msg_type = rspro_element.get_field('msg')
+        rspro_msg = rspro_element.get_field('msg_tree')
+        if msg_type == '12': # tpduModemToCard
+            modem2card = rspro_msg.get_field('tpduModemToCard_element')
+            #print(modem2card)
+            client_slot = modem2card.get_field('fromClientSlot_element')
+            csl = self.get_client_slot(client_slot)
+            bank_slot = modem2card.get_field('toBankSlot_element')
+            bsl = self.get_bank_slot(bank_slot)
+            self._set_or_verify_bank_slot(bsl)
+            data = modem2card.get_field('data').replace(':','')
+            logger.debug("C(%u:%u) -> B(%u:%u): %s" % (csl[0], csl[1], bsl[0], bsl[1], data))
+            # store the CMD portion until the RSP portion arrives later
+            self.cmd_tpdu = h2b(data)
+        elif msg_type == '13': # tpduCardToModem
+            card2modem = rspro_msg.get_field('tpduCardToModem_element')
+            #print(card2modem)
+            client_slot = card2modem.get_field('toClientSlot_element')
+            csl = self.get_client_slot(client_slot)
+            bank_slot = card2modem.get_field('fromBankSlot_element')
+            bsl = self.get_bank_slot(bank_slot)
+            self._set_or_verify_bank_slot(bsl)
+            data = card2modem.get_field('data').replace(':','')
+            logger.debug("C(%u:%u) <- B(%u:%u): %s" % (csl[0], csl[1], bsl[0], bsl[1], data))
+            rsp_tpdu = h2b(data)
+            if self.cmd_tpdu:
+                # combine this R-TPDU with the C-TPDU we saw earlier
+                r = Tpdu(self.cmd_tpdu, rsp_tpdu)
+                self.cmd_tpdu = False
+                return r
+        elif msg_type == '14': # clientSlotStatus
+            cl_slotstatus = rspro_msg.get_field('clientSlotStatusInd_element')
+            #print(cl_slotstatus)
+            client_slot = cl_slotstatus.get_field('fromClientSlot_element')
+            bank_slot = cl_slotstatus.get_field('toBankSlot_element')
+            slot_pstatus = cl_slotstatus.get_field('slotPhysStatus_element')
+            vccPresent, resetActive, clkActive = self.get_pstatus(slot_pstatus)
+            if vccPresent and clkActive and not resetActive:
+                logger.debug("RESET")
+                return CardReset()
+        else:
+            print("Unhandled msg type %s: %s" % (msg_type, rspro_msg))
+
+
+class PysharkRsproPcap(_PysharkRspro):
+    """APDU Source [provider] class for reading RSPRO (osmo-remsim) from a PCAP
+    file via pyshark, which in turn uses tshark (part of wireshark).
+
+    In order to use this, you need a wireshark patched with RSPRO support,
+    such as can be found at https://gitea.osmocom.org/osmocom/wireshark/src/branch/laforge/rspro
+
+    A STANDARD UPSTREAM WIRESHARK *DOES NOT WORK*.
+    """
+    def __init__(self, pcap_filename):
+        """
+        Args:
+            pcap_filename: File name of the pcap file to be opened
+        """
+        pyshark_inst = pyshark.FileCapture(pcap_filename, display_filter='rspro', use_json=True, keep_packets=False)
+        super().__init__(pyshark_inst)
+
+class PysharkRsproLive(_PysharkRspro):
+    """APDU Source [provider] class for reading RSPRO (osmo-remsim) from a live capture
+    via pyshark, which in turn uses tshark (part of wireshark).
+
+    In order to use this, you need a wireshark patched with RSPRO support,
+    such as can be found at https://gitea.osmocom.org/osmocom/wireshark/src/branch/laforge/rspro
+
+    A STANDARD UPSTREAM WIRESHARK *DOES NOT WORK*.
+    """
+    def __init__(self, interface, bpf_filter='tcp port 9999 or tcp port 9998'):
+        """
+        Args:
+            interface: Network interface name to capture packets on (like "eth0")
+            bfp_filter: libpcap capture filter to use
+        """
+        pyshark_inst = pyshark.LiveCapture(interface=interface, display_filter='rspro', bpf_filter=bpf_filter,
+                                           use_json=True)
+        super().__init__(pyshark_inst)

pySim/ara_m.py

@@ -0,0 +1,412 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+Support for the Secure Element Access Control, specifically the ARA-M inside an UICC.
+"""
+
+#
+# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+from construct import *
+from construct import Optional as COptional
+from pySim.construct import *
+from pySim.filesystem import *
+from pySim.tlv import *
+
+# various BER-TLV encoded Data Objects (DOs)
+
+
+class AidRefDO(BER_TLV_IE, tag=0x4f):
+    # SEID v1.1 Table 6-3
+    _construct = HexAdapter(GreedyBytes)
+
+
+class AidRefEmptyDO(BER_TLV_IE, tag=0xc0):
+    # SEID v1.1 Table 6-3
+    pass
+
+
+class DevAppIdRefDO(BER_TLV_IE, tag=0xc1):
+    # SEID v1.1 Table 6-4
+    _construct = HexAdapter(GreedyBytes)
+
+
+class PkgRefDO(BER_TLV_IE, tag=0xca):
+    # Android UICC Carrier Privileges specific extension, see https://source.android.com/devices/tech/config/uicc
+    _construct = Struct('package_name_string'/GreedyString("ascii"))
+
+
+class RefDO(BER_TLV_IE, tag=0xe1, nested=[AidRefDO, AidRefEmptyDO, DevAppIdRefDO, PkgRefDO]):
+    # SEID v1.1 Table 6-5
+    pass
+
+
+class ApduArDO(BER_TLV_IE, tag=0xd0):
+    # SEID v1.1 Table 6-8
+    def _from_bytes(self, do: bytes):
+        if len(do) == 1:
+            if do[0] == 0x00:
+                self.decoded = {'generic_access_rule': 'never'}
+                return self.decoded
+            elif do[0] == 0x01:
+                self.decoded = {'generic_access_rule': 'always'}
+                return self.decoded
+            else:
+                return ValueError('Invalid 1-byte generic APDU access rule')
+        else:
+            if len(do) % 8:
+                return ValueError('Invalid non-modulo-8 length of APDU filter: %d' % len(do))
+            self.decoded['apdu_filter'] = []
+            offset = 0
+            while offset < len(do):
+                self.decoded['apdu_filter'] += {'header': b2h(do[offset:offset+4]),
+                                                'mask': b2h(do[offset+4:offset+8])}
+            self.decoded = res
+            return res
+
+    def _to_bytes(self):
+        if 'generic_access_rule' in self.decoded:
+            if self.decoded['generic_access_rule'] == 'never':
+                return b'\x00'
+            elif self.decoded['generic_access_rule'] == 'always':
+                return b'\x01'
+            else:
+                return ValueError('Invalid 1-byte generic APDU access rule')
+        else:
+            if not 'apdu_filter' in self.decoded:
+                return ValueError('Invalid APDU AR DO')
+            filters = self.decoded['apdu_filter']
+            res = b''
+            for f in filters:
+                if not 'header' in f or not 'mask' in f:
+                    return ValueError('APDU filter must contain header and mask')
+                header_b = h2b(f['header'])
+                mask_b = h2b(f['mask'])
+                if len(header_b) != 4 or len(mask_b) != 4:
+                    return ValueError('APDU filter header and mask must each be 4 bytes')
+                res += header_b + mask_b
+            return res
+
+
+class NfcArDO(BER_TLV_IE, tag=0xd1):
+    # SEID v1.1 Table 6-9
+    _construct = Struct('nfc_event_access_rule' /
+                        Enum(Int8ub, never=0, always=1))
+
+
+class PermArDO(BER_TLV_IE, tag=0xdb):
+    # Android UICC Carrier Privileges specific extension, see https://source.android.com/devices/tech/config/uicc
+    _construct = Struct('permissions'/HexAdapter(Bytes(8)))
+
+
+class ArDO(BER_TLV_IE, tag=0xe3, nested=[ApduArDO, NfcArDO, PermArDO]):
+    # SEID v1.1 Table 6-7
+    pass
+
+
+class RefArDO(BER_TLV_IE, tag=0xe2, nested=[RefDO, ArDO]):
+    # SEID v1.1 Table 6-6
+    pass
+
+
+class ResponseAllRefArDO(BER_TLV_IE, tag=0xff40, nested=[RefArDO]):
+    # SEID v1.1 Table 4-2
+    pass
+
+
+class ResponseArDO(BER_TLV_IE, tag=0xff50, nested=[ArDO]):
+    # SEID v1.1 Table 4-3
+    pass
+
+
+class ResponseRefreshTagDO(BER_TLV_IE, tag=0xdf20):
+    # SEID v1.1 Table 4-4
+    _construct = Struct('refresh_tag'/HexAdapter(Bytes(8)))
+
+
+class DeviceInterfaceVersionDO(BER_TLV_IE, tag=0xe6):
+    # SEID v1.1 Table 6-12
+    _construct = Struct('major'/Int8ub, 'minor'/Int8ub, 'patch'/Int8ub)
+
+
+class DeviceConfigDO(BER_TLV_IE, tag=0xe4, nested=[DeviceInterfaceVersionDO]):
+    # SEID v1.1 Table 6-10
+    pass
+
+
+class ResponseDeviceConfigDO(BER_TLV_IE, tag=0xff7f, nested=[DeviceConfigDO]):
+    # SEID v1.1 Table 5-14
+    pass
+
+
+class AramConfigDO(BER_TLV_IE, tag=0xe5, nested=[DeviceInterfaceVersionDO]):
+    # SEID v1.1 Table 6-11
+    pass
+
+
+class ResponseAramConfigDO(BER_TLV_IE, tag=0xdf21, nested=[AramConfigDO]):
+    # SEID v1.1 Table 4-5
+    pass
+
+
+class CommandStoreRefArDO(BER_TLV_IE, tag=0xf0, nested=[RefArDO]):
+    # SEID v1.1 Table 5-2
+    pass
+
+
+class CommandDelete(BER_TLV_IE, tag=0xf1, nested=[AidRefDO, AidRefEmptyDO, RefDO, RefArDO]):
+    # SEID v1.1 Table 5-4
+    pass
+
+
+class CommandUpdateRefreshTagDO(BER_TLV_IE, tag=0xf2):
+    # SEID V1.1 Table 5-6
+    pass
+
+
+class CommandRegisterClientAidsDO(BER_TLV_IE, tag=0xf7, nested=[AidRefDO, AidRefEmptyDO]):
+    # SEID v1.1 Table 5-7
+    pass
+
+
+class CommandGet(BER_TLV_IE, tag=0xf3, nested=[AidRefDO, AidRefEmptyDO]):
+    # SEID v1.1 Table 5-8
+    pass
+
+
+class CommandGetAll(BER_TLV_IE, tag=0xf4):
+    # SEID v1.1 Table 5-9
+    pass
+
+
+class CommandGetClientAidsDO(BER_TLV_IE, tag=0xf6):
+    # SEID v1.1 Table 5-10
+    pass
+
+
+class CommandGetNext(BER_TLV_IE, tag=0xf5):
+    # SEID v1.1 Table 5-11
+    pass
+
+
+class CommandGetDeviceConfigDO(BER_TLV_IE, tag=0xf8):
+    # SEID v1.1 Table 5-12
+    pass
+
+
+class ResponseAracAidDO(BER_TLV_IE, tag=0xff70, nested=[AidRefDO, AidRefEmptyDO]):
+    # SEID v1.1 Table 5-13
+    pass
+
+
+class BlockDO(BER_TLV_IE, tag=0xe7):
+    # SEID v1.1 Table 6-13
+    _construct = Struct('offset'/Int16ub, 'length'/Int8ub)
+
+
+# SEID v1.1 Table 4-1
+class GetCommandDoCollection(TLV_IE_Collection, nested=[RefDO, DeviceConfigDO]):
+    pass
+
+# SEID v1.1 Table 4-2
+
+
+class GetResponseDoCollection(TLV_IE_Collection, nested=[ResponseAllRefArDO, ResponseArDO,
+                                                         ResponseRefreshTagDO, ResponseAramConfigDO]):
+    pass
+
+# SEID v1.1 Table 5-1
+
+
+class StoreCommandDoCollection(TLV_IE_Collection,
+                               nested=[BlockDO, CommandStoreRefArDO, CommandDelete,
+                                       CommandUpdateRefreshTagDO, CommandRegisterClientAidsDO,
+                                       CommandGet, CommandGetAll, CommandGetClientAidsDO,
+                                       CommandGetNext, CommandGetDeviceConfigDO]):
+    pass
+
+
+# SEID v1.1 Section 5.1.2
+class StoreResponseDoCollection(TLV_IE_Collection,
+                                nested=[ResponseAllRefArDO, ResponseAracAidDO, ResponseDeviceConfigDO]):
+    pass
+
+
+class ADF_ARAM(CardADF):
+    def __init__(self, aid='a00000015141434c00', name='ADF.ARA-M', fid=None, sfid=None,
+                 desc='ARA-M Application'):
+        super().__init__(aid=aid, fid=fid, sfid=sfid, name=name, desc=desc)
+        self.shell_commands += [self.AddlShellCommands()]
+        files = []
+        self.add_files(files)
+
+    @staticmethod
+    def xceive_apdu_tlv(tp, hdr: Hexstr, cmd_do, resp_cls, exp_sw='9000'):
+        """Transceive an APDU with the card, transparently encoding the command data from TLV
+        and decoding the response data tlv."""
+        if cmd_do:
+            cmd_do_enc = cmd_do.to_ie()
+            cmd_do_len = len(cmd_do_enc)
+            if cmd_do_len > 255:
+                return ValueError('DO > 255 bytes not supported yet')
+        else:
+            cmd_do_enc = b''
+            cmd_do_len = 0
+        c_apdu = hdr + ('%02x' % cmd_do_len) + b2h(cmd_do_enc)
+        (data, sw) = tp.send_apdu_checksw(c_apdu, exp_sw)
+        if data:
+            if resp_cls:
+                resp_do = resp_cls()
+                resp_do.from_tlv(h2b(data))
+                return resp_do
+            else:
+                return data
+        else:
+            return None
+
+    @staticmethod
+    def store_data(tp, do) -> bytes:
+        """Build the Command APDU for STORE DATA."""
+        return ADF_ARAM.xceive_apdu_tlv(tp, '80e29000', do, StoreResponseDoCollection)
+
+    @staticmethod
+    def get_all(tp):
+        return ADF_ARAM.xceive_apdu_tlv(tp, '80caff40', None, GetResponseDoCollection)
+
+    @staticmethod
+    def get_config(tp, v_major=0, v_minor=0, v_patch=1):
+        cmd_do = DeviceConfigDO()
+        cmd_do.from_dict([{'DeviceInterfaceVersionDO': {
+                         'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
+        return ADF_ARAM.xceive_apdu_tlv(tp, '80cadf21', cmd_do, ResponseAramConfigDO)
+
+    @with_default_category('Application-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        def __init(self):
+            super().__init__()
+
+        def do_aram_get_all(self, opts):
+            """GET DATA [All] on the ARA-M Applet"""
+            res_do = ADF_ARAM.get_all(self._cmd.card._scc._tp)
+            if res_do:
+                self._cmd.poutput_json(res_do.to_dict())
+
+        def do_aram_get_config(self, opts):
+            """Perform GET DATA [Config] on the ARA-M Applet: Tell it our version and retrieve its version."""
+            res_do = ADF_ARAM.get_config(self._cmd.card._scc._tp)
+            if res_do:
+                self._cmd.poutput_json(res_do.to_dict())
+
+        store_ref_ar_do_parse = argparse.ArgumentParser()
+        # REF-DO
+        store_ref_ar_do_parse.add_argument(
+            '--device-app-id', required=True, help='Identifies the specific device application that the rule appplies to. Hash of Certificate of Application Provider, or UUID. (20/32 hex bytes)')
+        aid_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
+        aid_grp.add_argument(
+            '--aid', help='Identifies the specific SE application for which rules are to be stored. Can be a partial AID, containing for example only the RID.  (5-16 hex bytes)')
+        aid_grp.add_argument('--aid-empty', action='store_true',
+                             help='No specific SE application, applies to all applications')
+        store_ref_ar_do_parse.add_argument(
+            '--pkg-ref', help='Full Android Java package name (up to 127 chars ASCII)')
+        # AR-DO
+        apdu_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
+        apdu_grp.add_argument(
+            '--apdu-never', action='store_true', help='APDU access is not allowed')
+        apdu_grp.add_argument(
+            '--apdu-always', action='store_true', help='APDU access is allowed')
+        apdu_grp.add_argument(
+            '--apdu-filter', help='APDU filter: 4 byte CLA/INS/P1/P2 followed by 4 byte mask (8 hex bytes)')
+        nfc_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
+        nfc_grp.add_argument('--nfc-always', action='store_true',
+                             help='NFC event access is allowed')
+        nfc_grp.add_argument('--nfc-never', action='store_true',
+                             help='NFC event access is not allowed')
+        store_ref_ar_do_parse.add_argument(
+            '--android-permissions', help='Android UICC Carrier Privilege Permissions (8 hex bytes)')
+
+        @cmd2.with_argparser(store_ref_ar_do_parse)
+        def do_aram_store_ref_ar_do(self, opts):
+            """Perform STORE DATA [Command-Store-REF-AR-DO] to store a (new) access rule."""
+            # REF
+            ref_do_content = []
+            if opts.aid:
+                ref_do_content += [{'AidRefDO': opts.aid}]
+            elif opts.aid_empty:
+                ref_do_content += [{'AidRefEmptyDO': None}]
+            ref_do_content += [{'DevAppIdRefDO': opts.device_app_id}]
+            if opts.pkg_ref:
+                ref_do_content += [{'PkgRefDO': opts.pkg_ref}]
+            # AR
+            ar_do_content = []
+            if opts.apdu_never:
+                ar_do_content += [{'ApduArDO': {'generic_access_rule': 'never'}}]
+            elif opts.apdu_always:
+                ar_do_content += [{'ApduArDO': {'generic_access_rule': 'always'}}]
+            elif opts.apdu_filter:
+                # TODO: multiple filters
+                ar_do_content += [{'ApduArDO': {'apdu_filter': [opts.apdu_filter]}}]
+            if opts.nfc_always:
+                ar_do_content += [{'NfcArDO': {'nfc_event_access_rule': 'always'}}]
+            elif opts.nfc_never:
+                ar_do_content += [{'NfcArDO': {'nfc_event_access_rule': 'never'}}]
+            if opts.android_permissions:
+                ar_do_content += [{'PermArDO': {'permissions': opts.android_permissions}}]
+            d = [{'RefArDO': [{'RefDO': ref_do_content}, {'ArDO': ar_do_content}]}]
+            csrado = CommandStoreRefArDO()
+            csrado.from_dict(d)
+            res_do = ADF_ARAM.store_data(self._cmd.card._scc._tp, csrado)
+            if res_do:
+                self._cmd.poutput_json(res_do.to_dict())
+
+        def do_aram_delete_all(self, opts):
+            """Perform STORE DATA [Command-Delete[all]] to delete all access rules."""
+            deldo = CommandDelete()
+            res_do = ADF_ARAM.store_data(self._cmd.card._scc._tp, deldo)
+            if res_do:
+                self._cmd.poutput_json(res_do.to_dict())
+
+
+# SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
+sw_aram = {
+    'ARA-M': {
+        '6381': 'Rule successfully stored but an access rule already exists',
+        '6382': 'Rule successfully stored bu contained at least one unknown (discarded) BER-TLV',
+        '6581': 'Memory Problem',
+        '6700': 'Wrong Length in Lc',
+        '6981': 'DO is not supported by the ARA-M/ARA-C',
+        '6982': 'Security status not satisfied',
+        '6984': 'Rules have been updated and must be read again / logical channels in use',
+        '6985': 'Conditions not satisfied',
+        '6a80': 'Incorrect values in the command data',
+        '6a84': 'Rules have been updated and must be read again',
+        '6a86': 'Incorrect P1 P2',
+        '6a88': 'Referenced data not found',
+        '6a89': 'Conflicting access rule already exists in the Secure Element',
+        '6d00': 'Invalid instruction',
+        '6e00': 'Invalid class',
+    }
+}
+
+
+class CardApplicationARAM(CardApplication):
+    def __init__(self):
+        super().__init__('ARA-M', adf=ADF_ARAM(), sw=sw_aram)

pySim/card_handler.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+
+""" pySim: card handler utilities.  A 'card handler' is some method
+by which cards can be inserted/removed into the card reader.  For
+normal smart card readers, this has to be done manually.  However,
+there are also automatic card feeders.
+"""
+
+#
+# (C) 2019 by Sysmocom s.f.m.c. GmbH
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from pySim.transport import LinkBase
+
+import subprocess
+import sys
+import yaml
+
+
+class CardHandlerBase:
+    """Abstract base class representing a mechanism for card insertion/removal."""
+
+    def __init__(self, sl: LinkBase):
+        self.sl = sl
+
+    def get(self, first: bool = False):
+        """Method called when pySim needs a new card to be inserted.
+
+        Args:
+                first : set to true when the get method is called the
+                        first time. This is required to prevent blocking
+                        when a card is already inserted into the reader.
+                        The reader API would not recognize that card as
+                        "new card" until it would be removed and re-inserted
+                        again.
+        """
+        print("Ready for Programming: ", end='')
+        self._get(first)
+
+    def error(self):
+        """Method called when pySim failed to program a card. Move card to 'bad' batch."""
+        print("Programming failed: ", end='')
+        self._error()
+
+    def done(self):
+        """Method called when pySim failed to program a card. Move card to 'good' batch."""
+        print("Programming successful: ", end='')
+        self._done()
+
+    def _get(self, first: bool = False):
+        pass
+
+    def _error(self):
+        pass
+
+    def _done(self):
+        pass
+
+
+class CardHandler(CardHandlerBase):
+    """Manual card handler: User is prompted to insert/remove card from the reader."""
+
+    def _get(self, first: bool = False):
+        print("Insert card now (or CTRL-C to cancel)")
+        self.sl.wait_for_card(newcardonly=not first)
+
+    def _error(self):
+        print("Remove card from reader")
+        print("")
+
+    def _done(self):
+        print("Remove card from reader")
+        print("")
+
+
+class CardHandlerAuto(CardHandlerBase):
+    """Automatic card handler: A machine is used to handle the cards."""
+
+    verbose = True
+
+    def __init__(self, sl: LinkBase, config_file: str):
+        super().__init__(sl)
+        print("Card handler Config-file: " + str(config_file))
+        with open(config_file) as cfg:
+            self.cmds = yaml.load(cfg, Loader=yaml.FullLoader)
+        self.verbose = (self.cmds.get('verbose') == True)
+
+    def __print_outout(self, out):
+        print("")
+        print("Card handler output:")
+        print("---------------------8<---------------------")
+        stdout = out[0].strip()
+        if len(stdout) > 0:
+            print("stdout:")
+            print(stdout)
+        stderr = out[1].strip()
+        if len(stderr) > 0:
+            print("stderr:")
+            print(stderr)
+        print("---------------------8<---------------------")
+        print("")
+
+    def __exec_cmd(self, command):
+        print("Card handler Commandline: " + str(command))
+
+        proc = subprocess.Popen(
+            [command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+        out = proc.communicate()
+        rc = proc.returncode
+
+        if rc != 0 or self.verbose:
+            self.__print_outout(out)
+
+        if rc != 0:
+            print("")
+            print("Error: Card handler failure! (rc=" + str(rc) + ")")
+            sys.exit(rc)
+
+    def _get(self, first: bool = False):
+        print("Transporting card into the reader-bay...")
+        self.__exec_cmd(self.cmds['get'])
+        if self.sl:
+            self.sl.connect()
+
+    def _error(self):
+        print("Transporting card to the error-bin...")
+        self.__exec_cmd(self.cmds['error'])
+        print("")
+
+    def _done(self):
+        print("Transporting card into the collector bin...")
+        self.__exec_cmd(self.cmds['done'])
+        print("")

pySim/card_key_provider.py

@@ -0,0 +1,174 @@
+# coding=utf-8
+"""Obtaining card parameters (mostly key data) from external source.
+
+This module contains a base class and a concrete implementation of
+obtaining card key material (or other card-individual parameters) from
+an external data source.
+
+This is used e.g. to keep PIN/PUK data in some file on disk, avoiding
+the need of manually entering the related card-individual data on every
+operation with pySim-shell.
+"""
+
+# (C) 2021 by Sysmocom s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Philipp Maier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import List, Dict, Optional
+
+import abc
+import csv
+
+card_key_providers = []  # type: List['CardKeyProvider']
+
+
+class CardKeyProvider(abc.ABC):
+    """Base class, not containing any concrete implementation."""
+
+    VALID_FIELD_NAMES = ['ICCID', 'ADM1',
+                         'IMSI', 'PIN1', 'PIN2', 'PUK1', 'PUK2']
+
+    # check input parameters, but do nothing concrete yet
+    def _verify_get_data(self, fields: List[str] = [], key: str = 'ICCID', value: str = "") -> Dict[str, str]:
+        """Verify multiple fields for identified card.
+
+        Args:
+                fields : list of valid field names such as 'ADM1', 'PIN1', ... which are to be obtained
+                key : look-up key to identify card data, such as 'ICCID'
+                value : value for look-up key to identify card data
+        Returns:
+                dictionary of {field, value} strings for each requested field from 'fields'
+        """
+        for f in fields:
+            if (f not in self.VALID_FIELD_NAMES):
+                raise ValueError("Requested field name '%s' is not a valid field name, valid field names are: %s" %
+                                 (f, str(self.VALID_FIELD_NAMES)))
+
+        if (key not in self.VALID_FIELD_NAMES):
+            raise ValueError("Key field name '%s' is not a valid field name, valid field names are: %s" %
+                             (key, str(self.VALID_FIELD_NAMES)))
+
+        return {}
+
+    def get_field(self, field: str, key: str = 'ICCID', value: str = "") -> Optional[str]:
+        """get a single field from CSV file using a specified key/value pair"""
+        fields = [field]
+        result = self.get(fields, key, value)
+        return result.get(field)
+
+    @abc.abstractmethod
+    def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
+        """Get multiple card-individual fields for identified card.
+
+        Args:
+                fields : list of valid field names such as 'ADM1', 'PIN1', ... which are to be obtained
+                key : look-up key to identify card data, such as 'ICCID'
+                value : value for look-up key to identify card data
+        Returns:
+                dictionary of {field, value} strings for each requested field from 'fields'
+        """
+
+
+class CardKeyProviderCsv(CardKeyProvider):
+    """Card key provider implementation that allows to query against a specified CSV file"""
+    csv_file = None
+    filename = None
+
+    def __init__(self, filename: str):
+        """
+        Args:
+                filename : file name (path) of CSV file containing card-individual key/data
+        """
+        self.csv_file = open(filename, 'r')
+        if not self.csv_file:
+            raise RuntimeError("Could not open CSV file '%s'" % filename)
+        self.filename = filename
+
+    def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
+        super()._verify_get_data(fields, key, value)
+
+        self.csv_file.seek(0)
+        cr = csv.DictReader(self.csv_file)
+        if not cr:
+            raise RuntimeError(
+                "Could not open DictReader for CSV-File '%s'" % self.filename)
+        cr.fieldnames = [field.upper() for field in cr.fieldnames]
+
+        rc = {}
+        for row in cr:
+            if row[key] == value:
+                for f in fields:
+                    if f in row:
+                        rc.update({f: row[f]})
+                    else:
+                        raise RuntimeError("CSV-File '%s' lacks column '%s'" %
+                                           (self.filename, f))
+        return rc
+
+
+def card_key_provider_register(provider: CardKeyProvider, provider_list=card_key_providers):
+    """Register a new card key provider.
+
+    Args:
+            provider : the to-be-registered provider
+            provider_list : override the list of providers from the global default
+    """
+    if not isinstance(provider, CardKeyProvider):
+        raise ValueError("provider is not a card data provier")
+    provider_list.append(provider)
+
+
+def card_key_provider_get(fields, key: str, value: str, provider_list=card_key_providers) -> Dict[str, str]:
+    """Query all registered card data providers for card-individual [key] data.
+
+    Args:
+            fields : list of valid field names such as 'ADM1', 'PIN1', ... which are to be obtained
+            key : look-up key to identify card data, such as 'ICCID'
+            value : value for look-up key to identify card data
+            provider_list : override the list of providers from the global default
+    Returns:
+            dictionary of {field, value} strings for each requested field from 'fields'
+    """
+    for p in provider_list:
+        if not isinstance(p, CardKeyProvider):
+            raise ValueError(
+                "provider list contains element which is not a card data provier")
+        result = p.get(fields, key, value)
+        if result:
+            return result
+    return {}
+
+
+def card_key_provider_get_field(field: str, key: str, value: str, provider_list=card_key_providers) -> Optional[str]:
+    """Query all registered card data providers for a single field.
+
+    Args:
+            field : name valid field such as 'ADM1', 'PIN1', ... which is to be obtained
+            key : look-up key to identify card data, such as 'ICCID'
+            value : value for look-up key to identify card data
+            provider_list : override the list of providers from the global default
+    Returns:
+            dictionary of {field, value} strings for the requested field
+    """
+    for p in provider_list:
+        if not isinstance(p, CardKeyProvider):
+            raise ValueError(
+                "provider list contains element which is not a card data provier")
+        result = p.get_field(field, key, value)
+        if result:
+            return result
+    return None

pySim/commands.py

@@ -0,0 +1,632 @@
+# -*- coding: utf-8 -*-
+
+""" pySim: SIM Card commands according to ISO 7816-4 and TS 11.11
+"""
+
+#
+# Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
+# Copyright (C) 2010-2021  Harald Welte <laforge@gnumonks.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from construct import *
+from pySim.construct import LV
+from pySim.utils import rpad, b2h, h2b, sw_match, bertlv_encode_len, Hexstr, h2i, str_sanitize, expand_hex
+from pySim.exceptions import SwMatchError
+
+
+class SimCardCommands:
+    def __init__(self, transport):
+        self._tp = transport
+        self.cla_byte = "a0"
+        self.sel_ctrl = "0000"
+
+    # Extract a single FCP item from TLV
+    def __parse_fcp(self, fcp):
+        # see also: ETSI TS 102 221, chapter 11.1.1.3.1 Response for MF,
+        # DF or ADF
+        from pytlv.TLV import TLV
+        tlvparser = TLV(['82', '83', '84', 'a5', '8a', '8b',
+                        '8c', '80', 'ab', 'c6', '81', '88'])
+
+        # pytlv is case sensitive!
+        fcp = fcp.lower()
+
+        if fcp[0:2] != '62':
+            raise ValueError(
+                'Tag of the FCP template does not match, expected 62 but got %s' % fcp[0:2])
+
+        # Unfortunately the spec is not very clear if the FCP length is
+        # coded as one or two byte vale, so we have to try it out by
+        # checking if the length of the remaining TLV string matches
+        # what we get in the length field.
+        # See also ETSI TS 102 221, chapter 11.1.1.3.0 Base coding.
+        exp_tlv_len = int(fcp[2:4], 16)
+        if len(fcp[4:]) // 2 == exp_tlv_len:
+            skip = 4
+        else:
+            exp_tlv_len = int(fcp[2:6], 16)
+            if len(fcp[4:]) // 2 == exp_tlv_len:
+                skip = 6
+
+        # Skip FCP tag and length
+        tlv = fcp[skip:]
+        return tlvparser.parse(tlv)
+
+    # Tell the length of a record by the card response
+    # USIMs respond with an FCP template, which is different
+    # from what SIMs responds. See also:
+    # USIM: ETSI TS 102 221, chapter 11.1.1.3 Response Data
+    # SIM: GSM 11.11, chapter 9.2.1 SELECT
+    def __record_len(self, r) -> int:
+        if self.sel_ctrl == "0004":
+            tlv_parsed = self.__parse_fcp(r[-1])
+            file_descriptor = tlv_parsed['82']
+            # See also ETSI TS 102 221, chapter 11.1.1.4.3 File Descriptor
+            return int(file_descriptor[4:8], 16)
+        else:
+            return int(r[-1][28:30], 16)
+
+    # Tell the length of a binary file. See also comment
+    # above.
+    def __len(self, r) -> int:
+        if self.sel_ctrl == "0004":
+            tlv_parsed = self.__parse_fcp(r[-1])
+            return int(tlv_parsed['80'], 16)
+        else:
+            return int(r[-1][4:8], 16)
+
+    def get_atr(self) -> str:
+        """Return the ATR of the currently inserted card."""
+        return self._tp.get_atr()
+
+    def try_select_path(self, dir_list):
+        """ Try to select a specified path
+
+        Args:
+                dir_list : list of hex-string FIDs
+        """
+
+        rv = []
+        if type(dir_list) is not list:
+            dir_list = [dir_list]
+        for i in dir_list:
+            data, sw = self._tp.send_apdu(
+                self.cla_byte + "a4" + self.sel_ctrl + "02" + i)
+            rv.append((data, sw))
+            if sw != '9000':
+                return rv
+        return rv
+
+    def select_path(self, dir_list):
+        """Execute SELECT for an entire list/path of FIDs.
+
+        Args:
+                dir_list: list of FIDs representing the path to select
+
+        Returns:
+                list of return values (FCP in hex encoding) for each element of the path
+        """
+        rv = []
+        if type(dir_list) is not list:
+            dir_list = [dir_list]
+        for i in dir_list:
+            data, sw = self.select_file(i)
+            rv.append(data)
+        return rv
+
+    def select_file(self, fid: str):
+        """Execute SELECT a given file by FID.
+
+        Args:
+                fid : file identifier as hex string
+        """
+
+        return self._tp.send_apdu_checksw(self.cla_byte + "a4" + self.sel_ctrl + "02" + fid)
+
+    def select_parent_df(self):
+        """Execute SELECT to switch to the parent DF """
+        return self._tp.send_apdu_checksw(self.cla_byte + "a4030400")
+
+    def select_adf(self, aid: str):
+        """Execute SELECT a given Applicaiton ADF.
+
+        Args:
+                aid : application identifier as hex string
+        """
+
+        aidlen = ("0" + format(len(aid) // 2, 'x'))[-2:]
+        return self._tp.send_apdu_checksw(self.cla_byte + "a4" + "0404" + aidlen + aid)
+
+    def read_binary(self, ef, length: int = None, offset: int = 0):
+        """Execute READD BINARY.
+
+        Args:
+                ef : string or list of strings indicating name or path of transparent EF
+                length : number of bytes to read
+                offset : byte offset in file from which to start reading
+        """
+        r = self.select_path(ef)
+        if len(r[-1]) == 0:
+            return (None, None)
+        if length is None:
+            length = self.__len(r) - offset
+        if length < 0:
+            return (None, None)
+
+        total_data = ''
+        chunk_offset = 0
+        while chunk_offset < length:
+            chunk_len = min(255, length-chunk_offset)
+            pdu = self.cla_byte + \
+                'b0%04x%02x' % (offset + chunk_offset, chunk_len)
+            try:
+                data, sw = self._tp.send_apdu_checksw(pdu)
+            except Exception as e:
+                raise ValueError('%s, failed to read (offset %d)' %
+                                 (str_sanitize(str(e)), offset))
+            total_data += data
+            chunk_offset += chunk_len
+        return total_data, sw
+
+    def update_binary(self, ef, data: str, offset: int = 0, verify: bool = False, conserve: bool = False):
+        """Execute UPDATE BINARY.
+
+        Args:
+                ef : string or list of strings indicating name or path of transparent EF
+                data : hex string of data to be written
+                offset : byte offset in file from which to start writing
+                verify : Whether or not to verify data after write
+        """
+
+        file_len = self.binary_size(ef)
+        data = expand_hex(data, file_len)
+
+        data_length = len(data) // 2
+
+        # Save write cycles by reading+comparing before write
+        if conserve:
+            data_current, sw = self.read_binary(ef, data_length, offset)
+            if data_current == data:
+                return None, sw
+
+        self.select_path(ef)
+        total_data = ''
+        chunk_offset = 0
+        while chunk_offset < data_length:
+            chunk_len = min(255, data_length - chunk_offset)
+            # chunk_offset is bytes, but data slicing is hex chars, so we need to multiply by 2
+            pdu = self.cla_byte + \
+                'd6%04x%02x' % (offset + chunk_offset, chunk_len) + \
+                data[chunk_offset*2: (chunk_offset+chunk_len)*2]
+            try:
+                chunk_data, chunk_sw = self._tp.send_apdu_checksw(pdu)
+            except Exception as e:
+                raise ValueError('%s, failed to write chunk (chunk_offset %d, chunk_len %d)' %
+                                 (str_sanitize(str(e)), chunk_offset, chunk_len))
+            total_data += data
+            chunk_offset += chunk_len
+        if verify:
+            self.verify_binary(ef, data, offset)
+        return total_data, chunk_sw
+
+    def verify_binary(self, ef, data: str, offset: int = 0):
+        """Verify contents of transparent EF.
+
+        Args:
+                ef : string or list of strings indicating name or path of transparent EF
+                data : hex string of expected data
+                offset : byte offset in file from which to start verifying
+        """
+        res = self.read_binary(ef, len(data) // 2, offset)
+        if res[0].lower() != data.lower():
+            raise ValueError('Binary verification failed (expected %s, got %s)' % (
+                data.lower(), res[0].lower()))
+
+    def read_record(self, ef, rec_no: int):
+        """Execute READ RECORD.
+
+        Args:
+                ef : string or list of strings indicating name or path of linear fixed EF
+                rec_no : record number to read
+        """
+        r = self.select_path(ef)
+        rec_length = self.__record_len(r)
+        pdu = self.cla_byte + 'b2%02x04%02x' % (rec_no, rec_length)
+        return self._tp.send_apdu_checksw(pdu)
+
+    def update_record(self, ef, rec_no: int, data: str, force_len: bool = False, verify: bool = False,
+                      conserve: bool = False):
+        """Execute UPDATE RECORD.
+
+        Args:
+                ef : string or list of strings indicating name or path of linear fixed EF
+                rec_no : record number to read
+                data : hex string of data to be written
+                force_len : enforce record length by using the actual data length
+                verify : verify data by re-reading the record
+                conserve : read record and compare it with data, skip write on match
+        """
+
+        res = self.select_path(ef)
+        rec_length = self.__record_len(res)
+        data = expand_hex(data, rec_length)
+
+        if force_len:
+            # enforce the record length by the actual length of the given data input
+            rec_length = len(data) // 2
+        else:
+            # make sure the input data is padded to the record length using 0xFF.
+            # In cases where the input data exceed we throw an exception.
+            if (len(data) // 2 > rec_length):
+                raise ValueError('Data length exceeds record length (expected max %d, got %d)' % (
+                    rec_length, len(data) // 2))
+            elif (len(data) // 2 < rec_length):
+                data = rpad(data, rec_length * 2)
+
+        # Save write cycles by reading+comparing before write
+        if conserve:
+            data_current, sw = self.read_record(ef, rec_no)
+            data_current = data_current[0:rec_length*2]
+            if data_current == data:
+                return None, sw
+
+        pdu = (self.cla_byte + 'dc%02x04%02x' % (rec_no, rec_length)) + data
+        res = self._tp.send_apdu_checksw(pdu)
+        if verify:
+            self.verify_record(ef, rec_no, data)
+        return res
+
+    def verify_record(self, ef, rec_no: int, data: str):
+        """Verify record against given data
+
+        Args:
+                ef : string or list of strings indicating name or path of linear fixed EF
+                rec_no : record number to read
+                data : hex string of data to be verified
+        """
+        res = self.read_record(ef, rec_no)
+        if res[0].lower() != data.lower():
+            raise ValueError('Record verification failed (expected %s, got %s)' % (
+                data.lower(), res[0].lower()))
+
+    def record_size(self, ef):
+        """Determine the record size of given file.
+
+        Args:
+                ef : string or list of strings indicating name or path of linear fixed EF
+        """
+        r = self.select_path(ef)
+        return self.__record_len(r)
+
+    def record_count(self, ef):
+        """Determine the number of records in given file.
+
+        Args:
+                ef : string or list of strings indicating name or path of linear fixed EF
+        """
+        r = self.select_path(ef)
+        return self.__len(r) // self.__record_len(r)
+
+    def binary_size(self, ef):
+        """Determine the size of given transparent file.
+
+        Args:
+                ef : string or list of strings indicating name or path of transparent EF
+        """
+        r = self.select_path(ef)
+        return self.__len(r)
+
+    # TS 102 221 Section 11.3.1 low-level helper
+    def _retrieve_data(self, tag: int, first: bool = True):
+        if first:
+            pdu = '80cb008001%02x' % (tag)
+        else:
+            pdu = '80cb000000'
+        return self._tp.send_apdu_checksw(pdu)
+
+    def retrieve_data(self, ef, tag: int):
+        """Execute RETRIEVE DATA, see also TS 102 221 Section 11.3.1.
+
+        Args
+                ef : string or list of strings indicating name or path of transparent EF
+                tag : BER-TLV Tag of value to be retrieved
+        """
+        r = self.select_path(ef)
+        if len(r[-1]) == 0:
+            return (None, None)
+        total_data = ''
+        # retrieve first block
+        data, sw = self._retrieve_data(tag, first=True)
+        total_data += data
+        while sw == '62f1' or sw == '62f2':
+            data, sw = self._retrieve_data(tag, first=False)
+            total_data += data
+        return total_data, sw
+
+    # TS 102 221 Section 11.3.2 low-level helper
+    def _set_data(self, data: str, first: bool = True):
+        if first:
+            p1 = 0x80
+        else:
+            p1 = 0x00
+        if isinstance(data, bytes) or isinstance(data, bytearray):
+            data = b2h(data)
+        pdu = '80db00%02x%02x%s' % (p1, len(data)//2, data)
+        return self._tp.send_apdu_checksw(pdu)
+
+    def set_data(self, ef, tag: int, value: str, verify: bool = False, conserve: bool = False):
+        """Execute SET DATA.
+
+        Args
+                ef : string or list of strings indicating name or path of transparent EF
+                tag : BER-TLV Tag of value to be stored
+                value : BER-TLV value to be stored
+        """
+        r = self.select_path(ef)
+        if len(r[-1]) == 0:
+            return (None, None)
+
+        # in case of deleting the data, we only have 'tag' but no 'value'
+        if not value:
+            return self._set_data('%02x' % tag, first=True)
+
+        # FIXME: proper BER-TLV encode
+        tl = '%02x%s' % (tag, b2h(bertlv_encode_len(len(value)//2)))
+        tlv = tl + value
+        tlv_bin = h2b(tlv)
+
+        first = True
+        total_len = len(tlv_bin)
+        remaining = tlv_bin
+        while len(remaining) > 0:
+            fragment = remaining[:255]
+            rdata, sw = self._set_data(fragment, first=first)
+            first = False
+            remaining = remaining[255:]
+        return rdata, sw
+
+    def run_gsm(self, rand: str):
+        """Execute RUN GSM ALGORITHM.
+
+        Args:
+                rand : 16 byte random data as hex string (RAND)
+        """
+        if len(rand) != 32:
+            raise ValueError('Invalid rand')
+        self.select_path(['3f00', '7f20'])
+        return self._tp.send_apdu(self.cla_byte + '88000010' + rand)
+
+    def authenticate(self, rand: str, autn: str, context='3g'):
+        """Execute AUTHENTICATE (USIM/ISIM).
+
+        Args:
+                rand : 16 byte random data as hex string (RAND)
+                autn : 8 byte Autentication Token (AUTN)
+                context : 16 byte random data ('3g' or 'gsm')
+        """
+        # 3GPP TS 31.102 Section 7.1.2.1
+        AuthCmd3G = Struct('rand'/LV, 'autn'/Optional(LV))
+        AuthResp3GSyncFail = Struct(Const(b'\xDC'), 'auts'/LV)
+        AuthResp3GSuccess = Struct(
+            Const(b'\xDB'), 'res'/LV, 'ck'/LV, 'ik'/LV, 'kc'/Optional(LV))
+        AuthResp3G = Select(AuthResp3GSyncFail, AuthResp3GSuccess)
+        # build parameters
+        cmd_data = {'rand': rand, 'autn': autn}
+        if context == '3g':
+            p2 = '81'
+        elif context == 'gsm':
+            p2 = '80'
+        (data, sw) = self._tp.send_apdu_constr_checksw(
+            self.cla_byte, '88', '00', p2, AuthCmd3G, cmd_data, AuthResp3G)
+        if 'auts' in data:
+            ret = {'synchronisation_failure': data}
+        else:
+            ret = {'successful_3g_authentication': data}
+        return (ret, sw)
+
+    def status(self):
+        """Execute a STATUS command as per TS 102 221 Section 11.1.2."""
+        return self._tp.send_apdu_checksw('80F20000ff')
+
+    def deactivate_file(self):
+        """Execute DECATIVATE FILE command as per TS 102 221 Section 11.1.14."""
+        return self._tp.send_apdu_constr_checksw(self.cla_byte, '04', '00', '00', None, None, None)
+
+    def activate_file(self, fid):
+        """Execute ACTIVATE FILE command as per TS 102 221 Section 11.1.15.
+
+        Args:
+                fid : file identifier as hex string
+        """
+        return self._tp.send_apdu_checksw(self.cla_byte + '44000002' + fid)
+
+    def create_file(self, payload: Hexstr):
+        """Execute CREEATE FILE command as per TS 102 222 Section 6.3"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e00000%02x%s' % (len(payload)//2, payload))
+
+    def delete_file(self, fid):
+        """Execute DELETE FILE command as per TS 102 222 Section 6.4"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e4000002' + fid)
+
+    def terminate_df(self, fid):
+        """Execute TERMINATE DF command as per TS 102 222 Section 6.7"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e6000002' + fid)
+
+    def terminate_ef(self, fid):
+        """Execute TERMINATE EF command as per TS 102 222 Section 6.8"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e8000002' + fid)
+
+    def terminate_card_usage(self):
+        """Execute TERMINATE CARD USAGE command as per TS 102 222 Section 6.9"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'fe000000')
+
+    def manage_channel(self, mode='open', lchan_nr=0):
+        """Execute MANAGE CHANNEL command as per TS 102 221 Section 11.1.17.
+
+        Args:
+                mode : logical channel operation code ('open' or 'close')
+                lchan_nr : logical channel number (1-19, 0=assigned by UICC)
+        """
+        if mode == 'close':
+            p1 = 0x80
+        else:
+            p1 = 0x00
+        pdu = self.cla_byte + '70%02x%02x00' % (p1, lchan_nr)
+        return self._tp.send_apdu_checksw(pdu)
+
+    def reset_card(self):
+        """Physically reset the card"""
+        return self._tp.reset_card()
+
+    def _chv_process_sw(self, op_name, chv_no, pin_code, sw):
+        if sw_match(sw, '63cx'):
+            raise RuntimeError('Failed to %s chv_no 0x%02X with code 0x%s, %i tries left.' %
+                               (op_name, chv_no, b2h(pin_code).upper(), int(sw[3])))
+        elif (sw != '9000'):
+            raise SwMatchError(sw, '9000')
+
+    def verify_chv(self, chv_no: int, code: str):
+        """Verify a given CHV (Card Holder Verification == PIN)
+
+        Args:
+                chv_no : chv number (1=CHV1, 2=CHV2, ...)
+                code : chv code as hex string
+        """
+        fc = rpad(b2h(code), 16)
+        data, sw = self._tp.send_apdu(
+            self.cla_byte + '2000' + ('%02X' % chv_no) + '08' + fc)
+        self._chv_process_sw('verify', chv_no, code, sw)
+        return (data, sw)
+
+    def unblock_chv(self, chv_no: int, puk_code: str, pin_code: str):
+        """Unblock a given CHV (Card Holder Verification == PIN)
+
+        Args:
+                chv_no : chv number (1=CHV1, 2=CHV2, ...)
+                puk_code : puk code as hex string
+                pin_code : new chv code as hex string
+        """
+        fc = rpad(b2h(puk_code), 16) + rpad(b2h(pin_code), 16)
+        data, sw = self._tp.send_apdu(
+            self.cla_byte + '2C00' + ('%02X' % chv_no) + '10' + fc)
+        self._chv_process_sw('unblock', chv_no, pin_code, sw)
+        return (data, sw)
+
+    def change_chv(self, chv_no: int, pin_code: str, new_pin_code: str):
+        """Change a given CHV (Card Holder Verification == PIN)
+
+        Args:
+                chv_no : chv number (1=CHV1, 2=CHV2, ...)
+                pin_code : current chv code as hex string
+                new_pin_code : new chv code as hex string
+        """
+        fc = rpad(b2h(pin_code), 16) + rpad(b2h(new_pin_code), 16)
+        data, sw = self._tp.send_apdu(
+            self.cla_byte + '2400' + ('%02X' % chv_no) + '10' + fc)
+        self._chv_process_sw('change', chv_no, pin_code, sw)
+        return (data, sw)
+
+    def disable_chv(self, chv_no: int, pin_code: str):
+        """Disable a given CHV (Card Holder Verification == PIN)
+
+        Args:
+                chv_no : chv number (1=CHV1, 2=CHV2, ...)
+                pin_code : current chv code as hex string
+                new_pin_code : new chv code as hex string
+        """
+        fc = rpad(b2h(pin_code), 16)
+        data, sw = self._tp.send_apdu(
+            self.cla_byte + '2600' + ('%02X' % chv_no) + '08' + fc)
+        self._chv_process_sw('disable', chv_no, pin_code, sw)
+        return (data, sw)
+
+    def enable_chv(self, chv_no: int, pin_code: str):
+        """Enable a given CHV (Card Holder Verification == PIN)
+
+        Args:
+                chv_no : chv number (1=CHV1, 2=CHV2, ...)
+                pin_code : chv code as hex string
+        """
+        fc = rpad(b2h(pin_code), 16)
+        data, sw = self._tp.send_apdu(
+            self.cla_byte + '2800' + ('%02X' % chv_no) + '08' + fc)
+        self._chv_process_sw('enable', chv_no, pin_code, sw)
+        return (data, sw)
+
+    def envelope(self, payload: str):
+        """Send one ENVELOPE command to the SIM
+
+        Args:
+                payload : payload as hex string
+        """
+        return self._tp.send_apdu_checksw('80c20000%02x%s' % (len(payload)//2, payload))
+
+    def terminal_profile(self, payload: str):
+        """Send TERMINAL PROFILE to card
+
+        Args:
+                payload : payload as hex string
+        """
+        data_length = len(payload) // 2
+        data, sw = self._tp.send_apdu(('80100000%02x' % data_length) + payload)
+        return (data, sw)
+
+    # ETSI TS 102 221 11.1.22
+    def suspend_uicc(self, min_len_secs: int = 60, max_len_secs: int = 43200):
+        """Send SUSPEND UICC to the card.
+
+        Args:
+                 min_len_secs : mimumum suspend time seconds
+                 max_len_secs : maximum suspend time seconds
+        """
+        def encode_duration(secs: int) -> Hexstr:
+            if secs >= 10*24*60*60:
+                return '04%02x' % (secs // (10*24*60*60))
+            elif secs >= 24*60*60:
+                return '03%02x' % (secs // (24*60*60))
+            elif secs >= 60*60:
+                return '02%02x' % (secs // (60*60))
+            elif secs >= 60:
+                return '01%02x' % (secs // 60)
+            else:
+                return '00%02x' % secs
+
+        def decode_duration(enc: Hexstr) -> int:
+            time_unit = enc[:2]
+            length = h2i(enc[2:4])
+            if time_unit == '04':
+                return length * 10*24*60*60
+            elif time_unit == '03':
+                return length * 24*60*60
+            elif time_unit == '02':
+                return length * 60*60
+            elif time_unit == '01':
+                return length * 60
+            elif time_unit == '00':
+                return length
+            else:
+                raise ValueError('Time unit must be 0x00..0x04')
+        min_dur_enc = encode_duration(min_len_secs)
+        max_dur_enc = encode_duration(max_len_secs)
+        data, sw = self._tp.send_apdu_checksw(
+            '8076000004' + min_dur_enc + max_dur_enc)
+        negotiated_duration_secs = decode_duration(data[:4])
+        resume_token = data[4:]
+        return (negotiated_duration_secs, resume_token, sw)
+
+    def get_data(self, tag: int, cla: int = 0x00):
+        data, sw = self._tp.send_apdu('%02xca%04x00' % (cla, tag))
+        return (data, sw)

pySim/construct.py

@@ -0,0 +1,265 @@
+from construct.lib.containers import Container, ListContainer
+from construct.core import EnumIntegerString
+import typing
+from construct import *
+from construct.core import evaluate, bytes2integer, integer2bytes, BitwisableString
+from construct.lib import integertypes
+from pySim.utils import b2h, h2b, swap_nibbles
+import gsm0338
+
+"""Utility code related to the integration of the 'construct' declarative parser."""
+
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class HexAdapter(Adapter):
+    """convert a bytes() type to a string of hex nibbles."""
+
+    def _decode(self, obj, context, path):
+        return b2h(obj)
+
+    def _encode(self, obj, context, path):
+        return h2b(obj)
+
+
+class BcdAdapter(Adapter):
+    """convert a bytes() type to a string of BCD nibbles."""
+
+    def _decode(self, obj, context, path):
+        return swap_nibbles(b2h(obj))
+
+    def _encode(self, obj, context, path):
+        return h2b(swap_nibbles(obj))
+
+class InvertAdapter(Adapter):
+    """inverse logic (false->true, true->false)."""
+    @staticmethod
+    def _invert_bool_in_obj(obj):
+        for k,v in obj.items():
+            # skip all private entries
+            if k.startswith('_'):
+                continue
+            if v == False:
+                obj[k] = True
+            elif v == True:
+                obj[k] = False
+        return obj
+
+    def _decode(self, obj, context, path):
+        return self._invert_bool_in_obj(obj)
+
+    def _encode(self, obj, context, path):
+        return self._invert_bool_in_obj(obj)
+
+class Rpad(Adapter):
+    """
+    Encoder appends padding bytes (b'\\xff') up to target size.
+    Decoder removes trailing padding bytes.
+
+    Parameters:
+        subcon: Subconstruct as defined by construct library
+        pattern: set padding pattern (default: b'\\xff')
+    """
+
+    def __init__(self, subcon, pattern=b'\xff'):
+        super().__init__(subcon)
+        self.pattern = pattern
+
+    def _decode(self, obj, context, path):
+        return obj.rstrip(self.pattern)
+
+    def _encode(self, obj, context, path):
+        if len(obj) > self.sizeof():
+            raise SizeofError("Input ({}) exceeds target size ({})".format(
+                len(obj), self.sizeof()))
+        return obj + self.pattern * (self.sizeof() - len(obj))
+
+
+class GsmStringAdapter(Adapter):
+    """Convert GSM 03.38 encoded bytes to a string."""
+
+    def __init__(self, subcon, codec='gsm03.38', err='strict'):
+        super().__init__(subcon)
+        self.codec = codec
+        self.err = err
+
+    def _decode(self, obj, context, path):
+        return obj.decode(self.codec)
+
+    def _encode(self, obj, context, path):
+        return obj.encode(self.codec, self.err)
+
+
+def filter_dict(d, exclude_prefix='_'):
+    """filter the input dict to ensure no keys starting with 'exclude_prefix' remain."""
+    if not isinstance(d, dict):
+        return d
+    res = {}
+    for (key, value) in d.items():
+        if key.startswith(exclude_prefix):
+            continue
+        if type(value) is dict:
+            res[key] = filter_dict(value)
+        else:
+            res[key] = value
+    return res
+
+
+def normalize_construct(c):
+    """Convert a construct specific type to a related base type, mostly useful
+    so we can serialize it."""
+    # we need to include the filter_dict as we otherwise get elements like this
+    # in the dict: '_io': <_io.BytesIO object at 0x7fdb64e05860> which we cannot json-serialize
+    c = filter_dict(c)
+    if isinstance(c, Container) or isinstance(c, dict):
+        r = {k: normalize_construct(v) for (k, v) in c.items()}
+    elif isinstance(c, ListContainer):
+        r = [normalize_construct(x) for x in c]
+    elif isinstance(c, list):
+        r = [normalize_construct(x) for x in c]
+    elif isinstance(c, EnumIntegerString):
+        r = str(c)
+    else:
+        r = c
+    return r
+
+
+def parse_construct(c, raw_bin_data: bytes, length: typing.Optional[int] = None, exclude_prefix: str = '_'):
+    """Helper function to wrap around normalize_construct() and filter_dict()."""
+    if not length:
+        length = len(raw_bin_data)
+    parsed = c.parse(raw_bin_data, total_len=length)
+    return normalize_construct(parsed)
+
+
+# here we collect some shared / common definitions of data types
+LV = Prefixed(Int8ub, HexAdapter(GreedyBytes))
+
+# Default value for Reserved for Future Use (RFU) bits/bytes
+# See TS 31.101 Sec. "3.4 Coding Conventions"
+__RFU_VALUE = 0
+
+# Field that packs Reserved for Future Use (RFU) bit
+FlagRFU = Default(Flag, __RFU_VALUE)
+
+# Field that packs Reserved for Future Use (RFU) byte
+ByteRFU = Default(Byte, __RFU_VALUE)
+
+# Field that packs all remaining Reserved for Future Use (RFU) bytes
+GreedyBytesRFU = Default(GreedyBytes, b'')
+
+
+def BitsRFU(n=1):
+    '''
+    Field that packs Reserved for Future Use (RFU) bit(s)
+    as defined in TS 31.101 Sec. "3.4 Coding Conventions"
+
+    Use this for (currently) unused/reserved bits whose contents
+    should be initialized automatically but should not be cleared
+    in the future or when restoring read data (unlike padding).
+
+    Parameters:
+        n (Integer): Number of bits (default: 1)
+    '''
+    return Default(BitsInteger(n), __RFU_VALUE)
+
+
+def BytesRFU(n=1):
+    '''
+    Field that packs Reserved for Future Use (RFU) byte(s)
+    as defined in TS 31.101 Sec. "3.4 Coding Conventions"
+
+    Use this for (currently) unused/reserved bytes whose contents
+    should be initialized automatically but should not be cleared
+    in the future or when restoring read data (unlike padding).
+
+    Parameters:
+        n (Integer): Number of bytes (default: 1)
+    '''
+    return Default(Bytes(n), __RFU_VALUE)
+
+
+def GsmString(n):
+    '''
+    GSM 03.38 encoded byte string of fixed length n.
+    Encoder appends padding bytes (b'\\xff') to maintain
+    length. Decoder removes those trailing bytes.
+
+    Exceptions are raised for invalid characters
+    and length excess.
+
+    Parameters:
+        n (Integer): Fixed length of the encoded byte string
+    '''
+    return GsmStringAdapter(Rpad(Bytes(n), pattern=b'\xff'), codec='gsm03.38')
+
+class GreedyInteger(Construct):
+    """A variable-length integer implementation, think of combining GrredyBytes with BytesInteger."""
+    def __init__(self, signed=False, swapped=False, minlen=0):
+        super().__init__()
+        self.signed = signed
+        self.swapped = swapped
+        self.minlen = minlen
+
+    def _parse(self, stream, context, path):
+        data = stream_read_entire(stream, path)
+        if evaluate(self.swapped, context):
+            data = swapbytes(data)
+        try:
+            return bytes2integer(data, self.signed)
+        except ValueError as e:
+            raise IntegerError(str(e), path=path)
+
+    def __bytes_required(self, i, minlen=0):
+        if self.signed:
+            raise NotImplementedError("FIXME: Implement support for encoding signed integer")
+
+        # compute how many bytes we need
+        nbytes = 1
+        while True:
+            i = i >> 8
+            if i == 0:
+                break
+            else:
+                nbytes = nbytes + 1
+
+        # round up to the minimum number
+        # of bytes we anticipate
+        if nbytes < minlen:
+            nbytes = minlen
+
+        return nbytes
+
+    def _build(self, obj, stream, context, path):
+        if not isinstance(obj, integertypes):
+            raise IntegerError(f"value {obj} is not an integer", path=path)
+        length = self.__bytes_required(obj, self.minlen)
+        try:
+            data = integer2bytes(obj, length, self.signed)
+        except ValueError as e:
+            raise IntegerError(str(e), path=path)
+        if evaluate(self.swapped, context):
+            data = swapbytes(data)
+        stream_write(stream, data, length, path)
+        return obj
+
+# merged definitions of 24.008 + 23.040
+TypeOfNumber = Enum(BitsInteger(3), unknown=0, international=1, national=2, network_specific=3,
+                    short_code=4, alphanumeric=5, abbreviated=6, reserved_for_extension=7)
+NumberingPlan = Enum(BitsInteger(4), unknown=0, isdn_e164=1, data_x121=3, telex_f69=4,
+                     sc_specific_5=5, sc_specific_6=6, national=8, private=9,
+                     ermes=10, reserved_cts=11, reserved_for_extension=15)
+TonNpi = BitStruct('ext'/Flag, 'type_of_number'/TypeOfNumber, 'numbering_plan_id'/NumberingPlan)

pySim/exceptions.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+""" pySim: Exceptions
+"""
+
+#
+# Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
+# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class NoCardError(Exception):
+    """No card was found in the reader."""
+    pass
+
+
+class ProtocolError(Exception):
+    """Some kind of protocol level error interfacing with the card."""
+    pass
+
+
+class ReaderError(Exception):
+    """Some kind of general error with the card reader."""
+    pass
+
+
+class SwMatchError(Exception):
+    """Raised when an operation specifies an expected SW but the actual SW from
+       the card doesn't match."""
+
+    def __init__(self, sw_actual: str, sw_expected: str, rs=None):
+        """
+        Args:
+                sw_actual : the SW we actually received from the card (4 hex digits)
+                sw_expected : the SW we expected to receive from the card (4 hex digits)
+                rs : interpreter class to convert SW to string
+        """
+        self.sw_actual = sw_actual
+        self.sw_expected = sw_expected
+        self.rs = rs
+
+    def __str__(self):
+        if self.rs and self.rs.lchan[0]:
+            r = self.rs.lchan[0].interpret_sw(self.sw_actual)
+            if r:
+                return "SW match failed! Expected %s and got %s: %s - %s" % (self.sw_expected, self.sw_actual, r[0], r[1])
+        return "SW match failed! Expected %s and got %s." % (self.sw_expected, self.sw_actual)

pySim/global_platform.py

@@ -0,0 +1,256 @@
+# coding=utf-8
+"""Partial Support for GlobalPLatform Card Spec (currently 2.1.1)
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from typing import Optional, List, Dict, Tuple
+from construct import Optional as COptional
+from construct import *
+from bidict import bidict
+from pySim.construct import *
+from pySim.utils import *
+from pySim.filesystem import *
+from pySim.tlv import *
+from pySim.profile import CardProfile
+
+sw_table = {
+    'Warnings': {
+        '6200': 'Logical Channel already closed',
+        '6283': 'Card Life Cycle State is CARD_LOCKED',
+        '6310': 'More data available',
+    },
+    'Execution errors': {
+        '6400': 'No specific diagnosis',
+        '6581': 'Memory failure',
+    },
+    'Checking errors': {
+        '6700': 'Wrong length in Lc',
+    },
+    'Functions in CLA not supported': {
+        '6881': 'Logical channel not supported or active',
+        '6882': 'Secure messaging not supported',
+    },
+    'Command not allowed': {
+        '6982': 'Security Status not satisfied',
+        '6985': 'Conditions of use not satisfied',
+    },
+    'Wrong parameters': {
+        '6a80': 'Incorrect values in command data',
+        '6a81': 'Function not supported e.g. card Life Cycle State is CARD_LOCKED',
+        '6a82': 'Application not found',
+        '6a84': 'Not enough memory space',
+        '6a86': 'Incorrect P1 P2',
+        '6a88': 'Referenced data not found',
+    },
+    'GlobalPlatform': {
+        '6d00': 'Invalid instruction',
+        '6e00': 'Invalid class',
+    },
+    'Application errors': {
+        '9484': 'Algorithm not supported',
+        '9485': 'Invalid key check value',
+    },
+}
+
+# GlobalPlatform 2.1.1 Section 9.1.6
+KeyType = Enum(Byte,    des=0x80,
+                        rsa_public_exponent_e_cleartex=0xA0,
+                        rsa_modulus_n_cleartext=0xA1,
+                        rsa_modulus_n=0xA2,
+                        rsa_private_exponent_d=0xA3,
+                        rsa_chines_remainder_p=0xA4,
+                        rsa_chines_remainder_q=0xA5,
+                        rsa_chines_remainder_pq=0xA6,
+                        rsa_chines_remainder_dpi=0xA7,
+                        rsa_chines_remainder_dqi=0xA8,
+                        not_available=0xff)
+
+# GlobalPlatform 2.1.1 Section 9.3.3.1
+# example:
+#   e0 48
+#       c0 04 01708010
+#       c0 04 02708010
+#       c0 04 03708010
+#       c0 04 01018010
+#       c0 04 02018010
+#       c0 04 03018010
+#       c0 04 01028010
+#       c0 04 02028010
+#       c0 04 03028010
+#       c0 04 01038010
+#       c0 04 02038010
+#       c0 04 03038010
+class KeyInformationData(BER_TLV_IE, tag=0xc0):
+    _construct = Struct('key_identifier'/Byte, 'key_version_number'/Byte,
+                        'key_types'/GreedyRange(KeyType))
+class KeyInformation(BER_TLV_IE, tag=0xe0, nested=[KeyInformationData]):
+    pass
+
+# card data sample, returned in response to GET DATA (80ca006600):
+# 66 31
+#    73 2f
+#        06 07
+#            2a864886fc6b01
+#        60 0c
+#            06 0a
+#                2a864886fc6b02020101
+#        63 09
+#            06 07
+#                2a864886fc6b03
+#        64 0b
+#            06 09
+#                2a864886fc6b040215
+
+# GlobalPlatform 2.1.1 Table F-1
+class ObjectIdentifier(BER_TLV_IE, tag=0x06):
+    _construct = GreedyBytes
+class CardManagementTypeAndVersion(BER_TLV_IE, tag=0x60, nested=[ObjectIdentifier]):
+    pass
+class CardIdentificationScheme(BER_TLV_IE, tag=0x63, nested=[ObjectIdentifier]):
+    pass
+class SecureChannelProtocolOfISD(BER_TLV_IE, tag=0x64, nested=[ObjectIdentifier]):
+    pass
+class CardConfigurationDetails(BER_TLV_IE, tag=0x65):
+    _construct = GreedyBytes
+class CardChipDetails(BER_TLV_IE, tag=0x66):
+    _construct = GreedyBytes
+class CardRecognitionData(BER_TLV_IE, tag=0x73, nested=[ObjectIdentifier,
+                                                        CardManagementTypeAndVersion,
+                                                        CardIdentificationScheme,
+                                                        SecureChannelProtocolOfISD,
+                                                        CardConfigurationDetails,
+                                                        CardChipDetails]):
+    pass
+class CardData(BER_TLV_IE, tag=0x66, nested=[CardRecognitionData]):
+    pass
+
+# GlobalPlatform 2.1.1 Table F-2
+class SecureChannelProtocolOfSelectedSD(BER_TLV_IE, tag=0x64, nested=[ObjectIdentifier]):
+    pass
+class SecurityDomainMgmtData(BER_TLV_IE, tag=0x73, nested=[CardManagementTypeAndVersion,
+                                                           CardIdentificationScheme,
+                                                           SecureChannelProtocolOfSelectedSD,
+                                                           CardConfigurationDetails,
+                                                           CardChipDetails]):
+    pass
+
+# GlobalPlatform 2.1.1 Section 9.1.1
+IsdLifeCycleState = Enum(Byte, op_ready=0x01, initialized=0x07, secured=0x0f,
+                         card_locked = 0x7f, terminated=0xff)
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class ApplicationID(BER_TLV_IE, tag=0x84):
+    _construct = GreedyBytes
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class SecurityDomainManagementData(BER_TLV_IE, tag=0x73):
+    _construct = GreedyBytes
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class ApplicationProductionLifeCycleData(BER_TLV_IE, tag=0x9f6e):
+    _construct = GreedyBytes
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class MaximumLengthOfDataFieldInCommandMessage(BER_TLV_IE, tag=0x9f65):
+    _construct = GreedyInteger()
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class ProprietaryData(BER_TLV_IE, tag=0xA5, nested=[SecurityDomainManagementData,
+                                                    ApplicationProductionLifeCycleData,
+                                                    MaximumLengthOfDataFieldInCommandMessage]):
+    pass
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class FciTemplate(BER_TLV_IE, tag=0x6f, nested=[ApplicationID, SecurityDomainManagementData,
+                                                ApplicationProductionLifeCycleData,
+                                                MaximumLengthOfDataFieldInCommandMessage,
+                                                ProprietaryData]):
+    pass
+
+class IssuerIdentificationNumber(BER_TLV_IE, tag=0x42):
+    _construct = BcdAdapter(GreedyBytes)
+
+class CardImageNumber(BER_TLV_IE, tag=0x45):
+    _construct = BcdAdapter(GreedyBytes)
+
+class SequenceCounterOfDefaultKvn(BER_TLV_IE, tag=0xc1):
+    _construct = GreedyInteger()
+
+class ConfirmationCounter(BER_TLV_IE, tag=0xc2):
+    _construct = GreedyInteger()
+
+# Collection of all the data objects we can get from GET DATA
+class DataCollection(TLV_IE_Collection, nested=[IssuerIdentificationNumber,
+                                                CardImageNumber,
+                                                CardData,
+                                                KeyInformation,
+                                                SequenceCounterOfDefaultKvn,
+                                                ConfirmationCounter]):
+    pass
+
+def decode_select_response(resp_hex: str) -> object:
+    t = FciTemplate()
+    t.from_tlv(h2b(resp_hex))
+    d = t.to_dict()
+    return flatten_dict_lists(d['fci_template'])
+
+# Application Dedicated File of a Security Domain
+class ADF_SD(CardADF):
+    def __init__(self, aid: str, name: str, desc: str):
+        super().__init__(aid=aid, fid=None, sfid=None, name=name, desc=desc)
+        self.shell_commands += [self.AddlShellCommands()]
+
+    @staticmethod
+    def decode_select_response(res_hex: str) -> object:
+        return decode_select_response(res_hex)
+
+    @with_default_category('Application-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        def __init__(self):
+            super().__init__()
+
+        def do_get_data(self, opts):
+            tlv_cls_name = opts.arg_list[0]
+            tlv_cls = DataCollection().members_by_name[tlv_cls_name]
+            (data, sw) = self._cmd.card._scc.get_data(cla=0x80, tag=tlv_cls.tag)
+            ie = tlv_cls()
+            ie.from_tlv(h2b(data))
+            self._cmd.poutput_json(ie.to_dict())
+
+        def complete_get_data(self, text, line, begidx, endidx) -> List[str]:
+            #data_dict = {camel_to_snake(str(x.__name__)): x for x in DataCollection.possible_nested}
+            data_dict = {str(x.__name__): x for x in DataCollection.possible_nested}
+            index_dict = {1: data_dict}
+            return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+# Card Application of a Security Domain
+class CardApplicationSD(CardApplication):
+    def __init__(self, aid: str, name: str, desc: str):
+        super().__init__(name, adf=ADF_SD(aid, name, desc), sw=sw_table)
+
+# Card Application of Issuer Security Domain
+class CardApplicationISD(CardApplicationSD):
+    # FIXME: ISD AID is not static, but could be different. One can select the empty
+    # application using '00a4040000' and then parse the response FCI to get the ISD AID
+    def __init__(self, aid='a000000003000000'):
+        super().__init__(aid=aid, name='ADF.ISD', desc='Issuer Security Domain')
+
+#class CardProfileGlobalPlatform(CardProfile):
+#    ORDER = 23
+#
+#    def __init__(self, name='GlobalPlatform'):
+#        super().__init__(name, desc='GlobalPlatfomr 2.1.1', cla=['00','80','84'], sw=sw_table)

pySim/gsm_r.py

@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+The File (and its derived classes) uses the classes of pySim.filesystem in
+order to describe the files specified in UIC Reference P38 T 9001 5.0 "FFFIS for GSM-R SIM Cards"
+"""
+
+#
+# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+from pySim.utils import *
+#from pySim.tlv import *
+from struct import pack, unpack
+from construct import *
+from construct import Optional as COptional
+from pySim.construct import *
+import enum
+
+from pySim.filesystem import *
+import pySim.ts_102_221
+import pySim.ts_51_011
+
+######################################################################
+# DF.EIRENE (FFFIS for GSM-R SIM Cards)
+######################################################################
+
+
+class FuncNTypeAdapter(Adapter):
+    def _decode(self, obj, context, path):
+        bcd = swap_nibbles(b2h(obj))
+        last_digit = bcd[-1]
+        return {'functional_number': bcd[:-1],
+                'presentation_of_only_this_fn': last_digit & 4,
+                'permanent_fn': last_digit & 8}
+
+    def _encode(self, obj, context, path):
+        return 'FIXME'
+
+
+class EF_FN(LinFixedEF):
+    """Section 7.2"""
+
+    def __init__(self):
+        super().__init__(fid='6ff1', sfid=None, name='EF.EN',
+                         desc='Functional numbers', rec_len=(9, 9))
+        self._construct = Struct('functional_number_and_type'/FuncNTypeAdapter(Bytes(8)),
+                                 'list_number'/Int8ub)
+
+
+class PlConfAdapter(Adapter):
+    """Section 7.4.3"""
+
+    def _decode(self, obj, context, path):
+        num = int(obj) & 0x7
+        if num == 0:
+            return 'None'
+        elif num == 1:
+            return 4
+        elif num == 2:
+            return 3
+        elif num == 3:
+            return 2
+        elif num == 4:
+            return 1
+        elif num == 5:
+            return 0
+
+    def _encode(self, obj, context, path):
+        if obj == 'None':
+            return 0
+        obj = int(obj)
+        if obj == 4:
+            return 1
+        elif obj == 3:
+            return 2
+        elif obj == 2:
+            return 3
+        elif obj == 1:
+            return 4
+        elif obj == 0:
+            return 5
+
+
+class PlCallAdapter(Adapter):
+    """Section 7.4.12"""
+
+    def _decode(self, obj, context, path):
+        num = int(obj) & 0x7
+        if num == 0:
+            return 'None'
+        elif num == 1:
+            return 4
+        elif num == 2:
+            return 3
+        elif num == 3:
+            return 2
+        elif num == 4:
+            return 1
+        elif num == 5:
+            return 0
+        elif num == 6:
+            return 'B'
+        elif num == 7:
+            return 'A'
+
+    def _encode(self, obj, context, path):
+        if obj == 'None':
+            return 0
+        if obj == 4:
+            return 1
+        elif obj == 3:
+            return 2
+        elif obj == 2:
+            return 3
+        elif obj == 1:
+            return 4
+        elif obj == 0:
+            return 5
+        elif obj == 'B':
+            return 6
+        elif obj == 'A':
+            return 7
+
+
+NextTableType = Enum(Byte, decision=0xf0, predefined=0xf1,
+                     num_dial_digits=0xf2, ic=0xf3, empty=0xff)
+
+
+class EF_CallconfC(TransparentEF):
+    """Section 7.3"""
+
+    def __init__(self):
+        super().__init__(fid='6ff2', sfid=None, name='EF.CallconfC', size=(24, 24),
+                         desc='Call Configuration of emergency calls Configuration')
+        self._construct = Struct('pl_conf'/PlConfAdapter(Int8ub),
+                                 'conf_nr'/BcdAdapter(Bytes(8)),
+                                 'max_rand'/Int8ub,
+                                 'n_ack_max'/Int16ub,
+                                 'pl_ack'/PlCallAdapter(Int8ub),
+                                 'n_nested_max'/Int8ub,
+                                 'train_emergency_gid'/Int8ub,
+                                 'shunting_emergency_gid'/Int8ub,
+                                 'imei'/BcdAdapter(Bytes(8)))
+
+
+class EF_CallconfI(LinFixedEF):
+    """Section 7.5"""
+
+    def __init__(self):
+        super().__init__(fid='6ff3', sfid=None, name='EF.CallconfI', rec_len=(21, 21),
+                         desc='Call Configuration of emergency calls Information')
+        self._construct = Struct('t_dur'/Int24ub,
+                                 't_relcalc'/Int32ub,
+                                 'pl_call'/PlCallAdapter(Int8ub),
+                                 'cause' /
+                                 FlagsEnum(Int8ub, powered_off=1,
+                                           radio_link_error=2, user_command=5),
+                                 'gcr'/BcdAdapter(Bytes(4)),
+                                 'fnr'/BcdAdapter(Bytes(8)))
+
+
+class EF_Shunting(TransparentEF):
+    """Section 7.6"""
+
+    def __init__(self):
+        super().__init__(fid='6ff4', sfid=None,
+                         name='EF.Shunting', desc='Shunting', size=(8, 8))
+        self._construct = Struct('common_gid'/Int8ub,
+                                 'shunting_gid'/HexAdapter(Bytes(7)))
+
+
+class EF_GsmrPLMN(LinFixedEF):
+    """Section 7.7"""
+
+    def __init__(self):
+        super().__init__(fid='6ff5', sfid=None, name='EF.GsmrPLMN',
+                         desc='GSM-R network selection', rec_len=(9, 9))
+        self._construct = Struct('plmn'/BcdAdapter(Bytes(3)),
+                                 'class_of_network'/BitStruct('supported'/FlagsEnum(BitsInteger(5), vbs=1, vgcs=2, emlpp=4, fn=8, eirene=16),
+                                                              'preference'/BitsInteger(3)),
+                                 'ic_incoming_ref_tbl'/HexAdapter(Bytes(2)),
+                                 'outgoing_ref_tbl'/HexAdapter(Bytes(2)),
+                                 'ic_table_ref'/HexAdapter(Bytes(1)))
+
+
+class EF_IC(LinFixedEF):
+    """Section 7.8"""
+
+    def __init__(self):
+        super().__init__(fid='6f8d', sfid=None, name='EF.IC',
+                         desc='International Code', rec_len=(7, 7))
+        self._construct = Struct('next_table_type'/NextTableType,
+                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'ic_decision_value'/BcdAdapter(Bytes(2)),
+                                 'network_string_table_index'/Int8ub)
+
+
+class EF_NW(LinFixedEF):
+    """Section 7.9"""
+
+    def __init__(self):
+        super().__init__(fid='6f80', sfid=None, name='EF.NW',
+                         desc='Network Name', rec_len=(8, 8))
+        self._construct = GsmString(8)
+
+
+class EF_Switching(LinFixedEF):
+    """Section 8.4"""
+
+    def __init__(self, fid, name, desc):
+        super().__init__(fid=fid, sfid=None,
+                         name=name, desc=desc, rec_len=(6, 6))
+        self._construct = Struct('next_table_type'/NextTableType,
+                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'decision_value'/BcdAdapter(Bytes(2)),
+                                 'string_table_index'/Int8ub)
+
+
+class EF_Predefined(LinFixedEF):
+    """Section 8.5"""
+
+    def __init__(self, fid, name, desc):
+        super().__init__(fid=fid, sfid=None,
+                         name=name, desc=desc, rec_len=(3, 3))
+        # header and other records have different structure. WTF !?!
+        self._construct = Struct('next_table_type'/NextTableType,
+                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'predefined_value1'/HexAdapter(Bytes(2)),
+                                 'string_table_index1'/Int8ub)
+        # TODO: predefined value n, ...
+
+
+class EF_DialledVals(TransparentEF):
+    """Section 8.6"""
+
+    def __init__(self, fid, name, desc):
+        super().__init__(fid=fid, sfid=None, name=name, desc=desc, size=(4, 4))
+        self._construct = Struct('next_table_type'/NextTableType,
+                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'dialed_digits'/BcdAdapter(Bytes(1)))
+
+
+class DF_EIRENE(CardDF):
+    def __init__(self, fid='7fe0', name='DF.EIRENE', desc='GSM-R EIRENE'):
+        super().__init__(fid=fid, name=name, desc=desc)
+        files = [
+            # Section 7.1.6 / Table 10 EIRENE GSM EFs
+            EF_FN(),
+            EF_CallconfC(),
+            EF_CallconfI(),
+            EF_Shunting(),
+            EF_GsmrPLMN(),
+            EF_IC(),
+            EF_NW(),
+
+            # support of the numbering plan
+            EF_Switching(fid='6f8e', name='EF.CT', desc='Call Type'),
+            EF_Switching(fid='6f8f', name='EF.SC', desc='Short Code'),
+            EF_Predefined(fid='6f88', name='EF.FC', desc='Function Code'),
+            EF_Predefined(fid='6f89', name='EF.Service',
+                          desc='VGCS/VBS Service Code'),
+            EF_Predefined(fid='6f8a', name='EF.Call',
+                          desc='First digit of the group ID'),
+            EF_Predefined(fid='6f8b', name='EF.FctTeam',
+                          desc='Call Type 6 Team Type + Team member function'),
+            EF_Predefined(fid='6f92', name='EF.Controller',
+                          desc='Call Type 7 Controller function code'),
+            EF_Predefined(fid='6f8c', name='EF.Gateway',
+                          desc='Access to external networks'),
+            EF_DialledVals(fid='6f81', name='EF.5to8digits',
+                           desc='Call Type 2 User Identity Number length'),
+            EF_DialledVals(fid='6f82', name='EF.2digits',
+                           desc='2 digits input'),
+            EF_DialledVals(fid='6f83', name='EF.8digits',
+                           desc='8 digits input'),
+            EF_DialledVals(fid='6f84', name='EF.9digits',
+                           desc='9 digits input'),
+            EF_DialledVals(fid='6f85', name='EF.SSSSS',
+                           desc='Group call area input'),
+            EF_DialledVals(fid='6f86', name='EF.LLLLL',
+                           desc='Location number Call Type 6'),
+            EF_DialledVals(fid='6f91', name='EF.Location',
+                           desc='Location number Call Type 7'),
+            EF_DialledVals(fid='6f87', name='EF.FreeNumber',
+                           desc='Free Number Call Type 0 and 8'),
+        ]
+        self.add_files(files)

pySim/gsmtap.py

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+
+""" Osmocom GSMTAP python implementation.
+GSMTAP is a packet format used for conveying a number of different
+telecom-related protocol traces over UDP.
+"""
+
+#
+# Copyright (C) 2022  Harald Welte <laforge@gnumonks.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import socket
+from typing import List, Dict, Optional
+from construct import Optional as COptional
+from construct import *
+from pySim.construct import *
+
+# The root definition of GSMTAP can be found at
+# https://cgit.osmocom.org/cgit/libosmocore/tree/include/osmocom/core/gsmtap.h
+
+GSMTAP_UDP_PORT = 4729
+
+# GSMTAP_TYPE_*
+gsmtap_type_construct = Enum(Int8ub,
+                             gsm_um = 0x01,
+                             gsm_abis = 0x02,
+                             gsm_um_burst = 0x03,
+                             sim = 0x04,
+                             tetra_i1 = 0x05,
+                             tetra_i1_burst = 0x06,
+                             wimax_burst = 0x07,
+                             gprs_gb_llc = 0x08,
+                             gprs_gb_sndcp = 0x09,
+                             gmr1_um = 0x0a,
+                             umts_rlc_mac = 0x0b,
+                             umts_rrc = 0x0c,
+                             lte_rrc = 0x0d,
+                             lte_mac = 0x0e,
+                             lte_mac_framed = 0x0f,
+                             osmocore_log = 0x10,
+                             qc_diag = 0x11,
+                             lte_nas = 0x12,
+                             e1_t1 = 0x13)
+
+
+# TYPE_UM_BURST
+gsmtap_subtype_burst_construct = Enum(Int8ub,
+                                      unknown = 0x00,
+                                      fcch = 0x01,
+                                      partial_sch = 0x02,
+                                      sch = 0x03,
+                                      cts_sch = 0x04,
+                                      compact_sch = 0x05,
+                                      normal = 0x06,
+                                      dummy = 0x07,
+                                      access = 0x08,
+                                      none = 0x09)
+
+gsmtap_subtype_wimax_burst_construct = Enum(Int8ub,
+                                            cdma_code = 0x10,
+                                            fch = 0x11,
+                                            ffb = 0x12,
+                                            pdu = 0x13,
+                                            hack = 0x14,
+                                            phy_attributes = 0x15)
+
+# GSMTAP_CHANNEL_*
+gsmtap_subtype_um_construct = Enum(Int8ub,
+                                   unknown = 0x00,
+                                   bcch = 0x01,
+                                   ccch = 0x02,
+                                   rach = 0x03,
+                                   agch = 0x04,
+                                   pch = 0x05,
+                                   sdcch = 0x06,
+                                   sdcch4 = 0x07,
+                                   sdcch8 = 0x08,
+                                   facch_f = 0x09,
+                                   facch_h = 0x0a,
+                                   pacch = 0x0b,
+                                   cbch52 = 0x0c,
+                                   pdtch = 0x0d,
+                                   ptcch = 0x0e,
+                                   cbch51 = 0x0f,
+                                   voice_f = 0x10,
+                                   voice_h = 0x11)
+
+
+# GSMTAP_SIM_*
+gsmtap_subtype_sim_construct = Enum(Int8ub,
+                                    apdu = 0x00,
+                                    atr = 0x01,
+                                    pps_req = 0x02,
+                                    pps_rsp = 0x03,
+                                    tpdu_hdr = 0x04,
+                                    tpdu_cmd = 0x05,
+                                    tpdu_rsp = 0x06,
+                                    tpdu_sw = 0x07)
+
+gsmtap_subtype_tetra_construct = Enum(Int8ub,
+                                      bsch = 0x01,
+                                      aach = 0x02,
+                                      sch_hu = 0x03,
+                                      sch_hd = 0x04,
+                                      sch_f = 0x05,
+                                      bnch = 0x06,
+                                      stch = 0x07,
+                                      tch_f = 0x08,
+                                      dmo_sch_s = 0x09,
+                                      dmo_sch_h = 0x0a,
+                                      dmo_sch_f = 0x0b,
+                                      dmo_stch = 0x0c,
+                                      dmo_tch = 0x0d)
+
+gsmtap_subtype_gmr1_construct = Enum(Int8ub,
+                                     unknown = 0x00,
+                                     bcch = 0x01,
+                                     ccch = 0x02,
+                                     pch = 0x03,
+                                     agch = 0x04,
+                                     bach = 0x05,
+                                     rach = 0x06,
+                                     cbch = 0x07,
+                                     sdcch = 0x08,
+                                     tachh = 0x09,
+                                     gbch = 0x0a,
+                                     tch3 = 0x10,
+                                     tch6 = 0x14,
+                                     tch9 = 0x18)
+
+gsmtap_subtype_e1t1_construct = Enum(Int8ub,
+                                     lapd = 0x01,
+                                     fr = 0x02,
+                                     raw = 0x03,
+                                     trau16 = 0x04,
+                                     trau8 = 0x05)
+
+gsmtap_arfcn_construct = BitStruct('pcs'/Flag, 'uplink'/Flag, 'arfcn'/BitsInteger(14))
+
+gsmtap_hdr_construct = Struct('version'/Int8ub,
+                              'hdr_len'/Int8ub,
+                              'type'/gsmtap_type_construct,
+                              'timeslot'/Int8ub,
+                              'arfcn'/gsmtap_arfcn_construct,
+                              'signal_dbm'/Int8sb,
+                              'snr_db'/Int8sb,
+                              'frame_nr'/Int32ub,
+                              'sub_type'/Switch(this.type, {
+                                                'gsm_um': gsmtap_subtype_um_construct,
+                                                'gsm_um_burst': gsmtap_subtype_burst_construct,
+                                                'sim': gsmtap_subtype_sim_construct,
+                                                'tetra_i1': gsmtap_subtype_tetra_construct,
+                                                'tetra_i1_burst': gsmtap_subtype_tetra_construct,
+                                                'wimax_burst': gsmtap_subtype_wimax_burst_construct,
+                                                'gmr1_um': gsmtap_subtype_gmr1_construct,
+                                                'e1_t1': gsmtap_subtype_e1t1_construct,
+                                                }),
+                              'antenna_nr'/Int8ub,
+                              'sub_slot'/Int8ub,
+                              'res'/Int8ub,
+                              'body'/GreedyBytes)
+
+osmocore_log_ts_construct = Struct('sec'/Int32ub, 'usec'/Int32ub)
+osmocore_log_level_construct = Enum(Int8ub, debug=1, info=3, notice=5, error=7, fatal=8)
+gsmtap_osmocore_log_hdr_construct = Struct('ts'/osmocore_log_ts_construct,
+                                           'proc_name'/PaddedString(16, 'ascii'),
+                                           'pid'/Int32ub,
+                                           'level'/osmocore_log_level_construct,
+                                           Bytes(3),
+                                           'subsys'/PaddedString(16, 'ascii'),
+                                           'src_file'/Struct('name'/PaddedString(32, 'ascii'), 'line_nr'/Int32ub))
+
+
+class GsmtapMessage:
+    """Class whose objects represent a single GSMTAP message. Can encode and decode messages."""
+    def __init__(self, encoded = None):
+        self.encoded = encoded
+        self.decoded = None
+
+    def decode(self):
+        self.decoded = parse_construct(gsmtap_hdr_construct, self.encoded)
+        return self.decoded
+
+    def encode(self, decoded):
+        self.encoded = gsmtap_hdr_construct.build(decoded)
+        return self.encoded
+
+class GsmtapSource:
+    def __init__(self, bind_ip:str='127.0.0.1', bind_port:int=4729):
+        self.bind_ip = bind_ip
+        self.bind_port = bind_port
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.sock.bind((self.bind_ip, self.bind_port))
+
+    def read_packet(self) -> GsmtapMessage:
+        data, addr = self.sock.recvfrom(1024)
+        gsmtap_msg = GsmtapMessage(data)
+        gsmtap_msg.decode()
+        if gsmtap_msg.decoded['version'] != 0x02:
+            raise ValueError('Unknown GSMTAP version 0x%02x' % gsmtap_msg.decoded['version'])
+        return gsmtap_msg.decoded, addr

pySim/iso7816_4.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+"""Utilities / Functions related to ISO 7816-4
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from construct import *
+from pySim.construct import *
+from pySim.utils import *
+from pySim.filesystem import *
+from pySim.tlv import *
+
+# Table 91 + Section 8.2.1.2
+class ApplicationId(BER_TLV_IE, tag=0x4f):
+    _construct = GreedyBytes
+
+# Table 91
+class ApplicationLabel(BER_TLV_IE, tag=0x50):
+    _construct = GreedyBytes
+
+# Table 91 + Section 5.3.1.2
+class FileReference(BER_TLV_IE, tag=0x51):
+    _construct = GreedyBytes
+
+# Table 91
+class CommandApdu(BER_TLV_IE, tag=0x52):
+    _construct = GreedyBytes
+
+# Table 91
+class DiscretionaryData(BER_TLV_IE, tag=0x53):
+    _construct = GreedyBytes
+
+# Table 91
+class DiscretionaryTemplate(BER_TLV_IE, tag=0x73):
+    _construct = GreedyBytes
+
+# Table 91 + RFC1738 / RFC2396
+class URL(BER_TLV_IE, tag=0x5f50):
+    _construct = GreedyString('ascii')
+
+# Table 91
+class ApplicationRelatedDOSet(BER_TLV_IE, tag=0x61):
+    _construct = GreedyBytes
+
+# Section 8.2.1.3 Application Template
+class ApplicationTemplate(BER_TLV_IE, tag=0x61, nested=[ApplicationId, ApplicationLabel, FileReference,
+                          CommandApdu, DiscretionaryData, DiscretionaryTemplate, URL,
+                          ApplicationRelatedDOSet]):
+    pass

pySim/jsonpath.py

@@ -0,0 +1,49 @@
+# coding=utf-8
+import json
+import pprint
+import jsonpath_ng
+
+"""JSONpath utility functions as needed within pysim.
+
+As pySim-sell has the ability to represent SIM files as JSON strings,
+adding JSONpath allows us to conveniently modify individual sub-fields
+of a file or record in its JSON representation.
+"""
+
+# (C) 2021 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def js_path_find(js_dict, js_path):
+    """Find/Match a JSON path within a given JSON-serializable dict.
+    Args:
+        js_dict : JSON-serializable dict to operate on
+        js_path : JSONpath string
+    Returns: Result of the JSONpath expression
+    """
+    jsonpath_expr = jsonpath_ng.parse(js_path)
+    return jsonpath_expr.find(js_dict)
+
+
+def js_path_modify(js_dict, js_path, new_val):
+    """Find/Match a JSON path within a given JSON-serializable dict.
+    Args:
+        js_dict : JSON-serializable dict to operate on
+        js_path : JSONpath string
+        new_val : New value for field in js_dict at js_path
+    """
+    jsonpath_expr = jsonpath_ng.parse(js_path)
+    jsonpath_expr.find(js_dict)
+    jsonpath_expr.update(js_dict, new_val)

pySim/ota.py

@@ -0,0 +1,461 @@
+"""Code related to SIM/UICC OTA according to TS 102 225 + TS 31.115."""
+
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from pySim.construct import *
+from pySim.utils import b2h
+from pySim.sms import UserDataHeader
+from construct import *
+from bidict import bidict
+import zlib
+import abc
+import struct
+from typing import Optional
+
+# ETS TS 102 225 gives the general command structure and the dialects for CAT_TP, TCP/IP and HTTPS
+# 3GPP TS 31.115 gives the dialects for SMS-PP, SMS-CB, USSD and HTTP
+
+# CPI CPL CHI CHL SPI KIc KID TAR CNTR PCNTR RC/CC/DS data
+
+#                                           CAT_TP  TCP/IP          SMS
+# CPI                                       0x01    0x01            =IEIa=70,len=0
+# CHI                                       NULL    NULL            NULL
+# CPI, CPL and CHL included in RC/CC/DS     true    true
+# RPI                                       0x02    0x02            =IEIa=71,len=0
+# RHI                                       NULL    NULL
+# RPI, RPL and RHL included in RC/CC/DS     true    true
+# packet-id                             0-bf,ff     0-bf,ff
+# identification packet                     false   102 225 tbl 6
+
+# KVN 1..f; KI1=KIc, KI2=KID, KI3=DEK
+
+# TS 102 225 Table 5
+ota_status_codes = bidict({
+    0x00:   'PoR OK',
+    0x01:   'RC/CC/DS failed',
+    0x02:   'CNTR low',
+    0x03:   'CNTR high',
+    0x04:   'CNTR blocked',
+    0x05:   'Ciphering error',
+    0x06:   'Unidentified security error',
+    0x07:   'Insufficient memory',
+    0x08:   'more time',
+    0x09:   'TAR unknown',
+    0x0a:   'Insufficient security level',
+    0x0b:   'Actual Response in SMS-SUBMIT',    # 31.115
+    0x0c:   'Actual Response in USSD',          # 31.115
+})
+
+# ETSI TS 102 225 Table 5 + 3GPP TS 31.115 Section 7
+ResponseStatus = Enum(Int8ub, por_ok=0, rc_cc_ds_failed=1, cntr_low=2, cntr_high=3,
+                      cntr_blocked=4, ciphering_error=5, undefined_security_error=6,
+                      insufficient_memory=7, more_time_needed=8, tar_unknown=9,
+                      insufficient_security_level=0x0A,
+                      actual_response_sms_submit=0x0B,
+                      actual_response_ussd=0x0C)
+
+# ETSI TS 102 226 Section 5.1.2
+CompactRemoteResp = Struct('number_of_commands'/Int8ub,
+                           'last_status_word'/HexAdapter(Bytes(2)),
+                           'last_response_data'/HexAdapter(GreedyBytes))
+
+RC_CC_DS = Enum(BitsInteger(2), no_rc_cc_ds=0, rc=1, cc=2, ds=3)
+
+# TS 102 225 Section 5.1.1 + TS 31.115 Section 4.2
+SPI = BitStruct(  # first octet
+    Padding(3),
+    'counter'/Enum(BitsInteger(2), no_counter=0, counter_no_replay_or_seq=1,
+                   counter_must_be_higher=2, counter_must_be_lower=3),
+    'ciphering'/Flag,
+    'rc_cc_ds'/RC_CC_DS,
+    # second octet
+    Padding(2),
+    'por_in_submit'/Flag,
+    'por_shall_be_ciphered'/Flag,
+    'por_rc_cc_ds'/RC_CC_DS,
+    'por'/Enum(BitsInteger(2), no_por=0,
+               por_required=1, por_only_when_error=2)
+)
+
+# TS 102 225 Section 5.1.2
+KIC = BitStruct('key'/BitsInteger(4),
+                'algo'/Enum(BitsInteger(4), implicit=0, single_des=1, triple_des_cbc2=5, triple_des_cbc3=9,
+                            aes_cbc=2)
+                )
+
+# TS 102 225 Section 5.1.3.1
+KID_CC = BitStruct('key'/BitsInteger(4),
+                   'algo'/Enum(BitsInteger(4), implicit=0, single_des=1, triple_des_cbc2=5, triple_des_cbc3=9,
+                               aes_cmac=2)
+                   )
+
+# TS 102 225 Section 5.1.3.2
+KID_RC = BitStruct('key'/BitsInteger(4),
+                   'algo'/Enum(BitsInteger(4), implicit=0, crc16=1, crc32=5, proprietary=3)
+                   )
+
+SmsCommandPacket = Struct('cmd_pkt_len'/Int16ub,
+                          'cmd_hdr_len'/Int8ub,
+                          'spi'/SPI,
+                          'kic'/KIC,
+                          'kid'/Switch(this.spi.rc_cc_ds, {'cc': KID_CC, 'rc': KID_RC }),
+                          'tar'/Bytes(3),
+                          'secured_data'/GreedyBytes)
+
+class OtaKeyset:
+    """The OTA related data (key material, counter) to be used in encrypt/decrypt."""
+    def __init__(self, algo_crypt: str, kic_idx: int, kic: bytes,
+                 algo_auth: str, kid_idx: int, kid: bytes, cntr: int = 0):
+        self.algo_crypt = algo_crypt
+        self.kic = bytes(kic)
+        self.kic_idx = kic_idx
+        self.algo_auth = algo_auth
+        self.kid = bytes(kid)
+        self.kid_idx = kid_idx
+        self.cntr = cntr
+
+    @property
+    def auth(self):
+        """Return an instance of the matching OtaAlgoAuth."""
+        return OtaAlgoAuth.fromKeyset(self)
+
+    @property
+    def crypt(self):
+        """Return an instance of the matching OtaAlgoCrypt."""
+        return OtaAlgoCrypt.fromKeyset(self)
+
+class OtaCheckError(Exception):
+    pass
+
+class OtaDialect(abc.ABC):
+    """Base Class for OTA dialects such as SMS, BIP, ..."""
+
+    def _compute_sig_len(self, spi:SPI):
+        if spi['rc_cc_ds'] == 'no_rc_cc_ds':
+            return 0
+        elif spi['rc_cc_ds'] == 'rc': # CRC-32
+            return 4
+        elif spi['rc_cc_ds'] == 'cc': # Cryptographic Checksum (CC)
+            # TODO: this is not entirely correct, as in AES case it could be 4 or 8
+            return 8
+        else:
+            raise ValueError("Invalid rc_cc_ds: %s" % spi['rc_cc_ds'])
+
+    @abc.abstractmethod
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, apdu: bytes) -> bytes:
+        pass
+
+    @abc.abstractmethod
+    def decode_resp(self, otak: OtaKeyset, apdu: bytes) -> (object, Optional["CompactRemoteResp"]):
+        """Decode a response into a response packet and, if indicted (by a
+        response status of `"por_ok"`) a decoded response.
+
+        The response packet's common characteristics are not fully determined,
+        and (so far) completely proprietary per dialect."""
+        pass
+
+
+from Crypto.Cipher import DES, DES3, AES
+from Crypto.Hash import CMAC
+
+class OtaAlgo(abc.ABC):
+    iv = b'\x00\x00\x00\x00\x00\x00\x00\x00'
+    blocksize = None
+    enum_name = None
+
+    @staticmethod
+    def _get_padding(in_len: int, multiple: int, padding: int = 0):
+        """Return padding bytes towards multiple of N."""
+        if in_len % multiple == 0:
+            return b''
+        pad_cnt = multiple - (in_len % multiple)
+        return b'\x00' * pad_cnt
+
+    @staticmethod
+    def _pad_to_multiple(indat: bytes, multiple: int, padding: int = 0):
+        """Pad input bytes to multiple of N."""
+        return indat + OtaAlgo._get_padding(len(indat), multiple, padding)
+
+    def pad_to_blocksize(self, indat: bytes, padding: int = 0):
+        """Pad the given input data to multiple of the cipher block size."""
+        return self._pad_to_multiple(indat, self.blocksize, padding)
+
+    def __init__(self, otak: OtaKeyset):
+        self.otak = otak
+
+    def __str__(self):
+        return self.__class__.__name__
+
+class OtaAlgoCrypt(OtaAlgo, abc.ABC):
+    def __init__(self, otak: OtaKeyset):
+        if self.enum_name != otak.algo_crypt:
+            raise ValueError('Cannot use algorithm %s with key for %s' % (self.enum_name, otak.algo_crypt))
+        super().__init__(otak)
+
+    def encrypt(self, data:bytes) -> bytes:
+        """Encrypt given input bytes using the key material given in constructor."""
+        padded_data = self.pad_to_blocksize(data)
+        return self._encrypt(data)
+
+    def decrypt(self, data:bytes) -> bytes:
+        """Decrypt given input bytes using the key material given in constructor."""
+        return self._decrypt(data)
+
+    @abc.abstractmethod
+    def _encrypt(self, data:bytes) -> bytes:
+        """Actual implementation, to be implemented by derived class."""
+        pass
+
+    @abc.abstractmethod
+    def _decrypt(self, data:bytes) -> bytes:
+        """Actual implementation, to be implemented by derived class."""
+        pass
+
+    @classmethod
+    def fromKeyset(cls, otak: OtaKeyset) -> 'OtaAlgoCrypt':
+        """Resolve the class for the encryption algorithm of otak and instantiate it."""
+        for subc in cls.__subclasses__():
+            if subc.enum_name == otak.algo_crypt:
+                return subc(otak)
+        raise ValueError('No implementation for crypt algorithm %s' % otak.algo_auth)
+
+class OtaAlgoAuth(OtaAlgo, abc.ABC):
+    def __init__(self, otak: OtaKeyset):
+        if self.enum_name != otak.algo_auth:
+            raise ValueError('Cannot use algorithm %s with key for %s' % (self.enum_name, otak.algo_crypt))
+        super().__init__(otak)
+
+    def sign(self, data:bytes) -> bytes:
+        """Compute the CC/CR check bytes for the input data using key material
+        given in constructor."""
+        padded_data = self.pad_to_blocksize(data)
+        sig = self._sign(padded_data)
+        return sig
+
+    def check_sig(self, data:bytes, cc_received:bytes):
+        """Compute the CC/CR check bytes for the input data and compare against cc_received."""
+        cc = self.sign(data)
+        if cc_received != cc:
+            raise OtaCheckError('Received CC (%s) != Computed CC (%s)' % (b2h(cc_received), b2h(cc)))
+
+    @abc.abstractmethod
+    def _sign(self, data:bytes) -> bytes:
+        """Actual implementation, to be implemented by derived class."""
+        pass
+
+    @classmethod
+    def fromKeyset(cls, otak: OtaKeyset) -> 'OtaAlgoAuth':
+        """Resolve the class for the authentication algorithm of otak and instantiate it."""
+        for subc in cls.__subclasses__():
+            if subc.enum_name == otak.algo_auth:
+                return subc(otak)
+        raise ValueError('No implementation for auth algorithm %s' % otak.algo_auth)
+
+class OtaAlgoCryptDES(OtaAlgoCrypt):
+    """DES is insecure.  For backwards compatibility with pre-Rel8"""
+    name = 'DES'
+    enum_name = 'single_des'
+    blocksize = 8
+    def _encrypt(self, data:bytes) -> bytes:
+        cipher = DES.new(self.otak.kic, DES.MODE_CBC, self.iv)
+        return cipher.encrypt(data)
+
+    def _decrypt(self, data:bytes) -> bytes:
+        cipher = DES.new(self.otak.kic, DES.MODE_CBC, self.iv)
+        return cipher.decrypt(data)
+
+class OtaAlgoAuthDES(OtaAlgoAuth):
+    """DES is insecure.  For backwards compatibility with pre-Rel8"""
+    name = 'DES'
+    enum_name = 'single_des'
+    blocksize = 8
+    def _sign(self, data:bytes) -> bytes:
+        cipher = DES.new(self.otak.kid, DES.MODE_CBC, self.iv)
+        ciph = cipher.encrypt(data)
+        return ciph[len(ciph) - 8:]
+
+class OtaAlgoCryptDES3(OtaAlgoCrypt):
+    name = '3DES'
+    enum_name = 'triple_des_cbc2'
+    blocksize = 8
+    def _encrypt(self, data:bytes) -> bytes:
+        cipher = DES3.new(self.otak.kic, DES3.MODE_CBC, self.iv)
+        return cipher.encrypt(data)
+
+    def _decrypt(self, data:bytes) -> bytes:
+        cipher = DES3.new(self.otak.kic, DES3.MODE_CBC, self.iv)
+        return cipher.decrypt(data)
+
+class OtaAlgoAuthDES3(OtaAlgoAuth):
+    name = '3DES'
+    enum_name = 'triple_des_cbc2'
+    blocksize = 8
+    def _sign(self, data:bytes) -> bytes:
+        cipher = DES3.new(self.otak.kid, DES3.MODE_CBC, self.iv)
+        ciph = cipher.encrypt(data)
+        return ciph[len(ciph) - 8:]
+
+class OtaAlgoCryptAES(OtaAlgoCrypt):
+    name = 'AES'
+    enum_name = 'aes_cbc'
+    blocksize = 16 # TODO: is this needed?
+    def _encrypt(self, data:bytes) -> bytes:
+        cipher = AES.new(self.otak.kic, AES.MODE_CBC, self.iv)
+        return cipher.encrypt(data)
+
+    def _decrypt(self, data:bytes) -> bytes:
+        cipher = AES.new(self.otak.kic, AES.MODE_CBC, self.iv)
+        return cipher.decrypt(data)
+
+class OtaAlgoAuthAES(OtaAlgoAuth):
+    name = 'AES'
+    enum_name = 'aes_cmac'
+    blocksize = 16 # TODO: is this needed?
+    def _sign(self, data:bytes) -> bytes:
+        cmac = CMAC.new(self.otak.kid, ciphermod=AES, mac_len=8)
+        cmac.update(data)
+        ciph = cmac.digest()
+        return ciph[len(ciph) - 8:]
+
+
+
+class OtaDialectSms(OtaDialect):
+    """OTA dialect for SMS based transport, as described in 3GPP TS 31.115."""
+    SmsResponsePacket = Struct('rpl'/Int16ub,
+                               'rhl'/Int8ub,
+                               'tar'/Bytes(3),
+                               'cntr'/Bytes(5),
+                               'pcntr'/Int8ub,
+                               'response_status'/ResponseStatus,
+                               'cc_rc'/Bytes(this.rhl-10),
+                               'secured_data'/GreedyBytes)
+
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict, apdu: bytes) -> bytes:
+        # length of signature in octets
+        len_sig = self._compute_sig_len(spi)
+        pad_cnt = 0
+        if spi['ciphering']: # ciphering is requested
+            # append padding bytes to end up with blocksize
+            len_cipher = 6 + len_sig + len(apdu)
+            padding = otak.crypt._get_padding(len_cipher, otak.crypt.blocksize)
+            pad_cnt = len(padding)
+            apdu += padding
+
+        kic = {'key': otak.kic_idx, 'algo': otak.algo_crypt}
+        kid = {'key': otak.kid_idx, 'algo': otak.algo_auth}
+
+        # CHL = number of octets from (and including) SPI to the end of RC/CC/DS
+        # 13 == SPI(2) + KIc(1) + KId(1) + TAR(3) + CNTR(5) + PCNTR(1)
+        chl = 13 + len_sig
+
+        # CHL + SPI (+ KIC + KID)
+        c = Struct('chl'/Int8ub, 'spi'/SPI, 'kic'/KIC, 'kid'/KID_CC, 'tar'/Bytes(3))
+        part_head = c.build({'chl': chl, 'spi':spi, 'kic':kic, 'kid':kid, 'tar':tar})
+        #print("part_head: %s" % b2h(part_head))
+
+        # CNTR + PCNTR (CNTR not used)
+        part_cnt = otak.cntr.to_bytes(5, 'big') + pad_cnt.to_bytes(1, 'big')
+        #print("part_cnt: %s" % b2h(part_cnt))
+
+        envelope_data = part_head + part_cnt + apdu
+        #print("envelope_data: %s" % b2h(envelope_data))
+
+        # 2-byte CPL. CPL is part of RC/CC/CPI to end of secured data, including any padding for ciphering
+        # CPL from and including CPI to end of secured data, including any padding for ciphering
+        cpl = len(envelope_data) + len_sig
+        envelope_data = cpl.to_bytes(2, 'big') + envelope_data
+        #print("envelope_data with cpl: %s" % b2h(envelope_data))
+
+        if spi['rc_cc_ds'] == 'cc':
+            cc = otak.auth.sign(envelope_data)
+            envelope_data = part_cnt + cc + apdu
+        elif spi['rc_cc_ds'] == 'rc':
+            # CRC32
+            crc32 = zlib.crc32(envelope_data) & 0xffffffff
+            envelope_data = part_cnt + crc32.to_bytes(4, 'big') + apdu
+        elif spi['rc_cc_ds'] == 'no_rc_cc_ds':
+            envelope_data = part_cnt + apdu
+        else:
+            raise ValueError("Invalid rc_cc_ds: %s" % spi['rc_cc_ds'])
+
+        #print("envelope_data with sig: %s" % b2h(envelope_data))
+
+        # encrypt as needed
+        if spi['ciphering']: # ciphering is requested
+            ciph = otak.crypt.encrypt(envelope_data)
+            envelope_data = part_head + ciph
+            # prefix with another CPL
+            cpl = len(envelope_data)
+            envelope_data = cpl.to_bytes(2, 'big') + envelope_data
+        else:
+            envelope_data = part_head + envelope_data
+
+        #print("envelope_data: %s" % b2h(envelope_data))
+
+        return envelope_data
+
+    def decode_resp(self, otak: OtaKeyset, spi: dict, data: bytes) -> ("OtaDialectSms.SmsResponsePacket", Optional["CompactRemoteResp"]):
+        if isinstance(data, str):
+            data = h2b(data)
+        # plain-text POR:   027100000e0ab000110000000000000001612f
+        # UDHL RPI IEDLa  RPL  RHL TAR    CNTR       PCNTR STS
+        # 02   71  00     000e 0a  b00011 0000000000 00    00  01 612f
+        # POR with CC:      027100001612b000110000000000000055f47118381175fb01612f
+        # POR with CC+CIPH: 027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c
+        if data[0] != 0x02:
+            raise ValueError('Unexpected UDL=0x%02x' % data[0])
+        udhd, remainder = UserDataHeader.fromBytes(data)
+        if not udhd.has_ie(0x71):
+            raise ValueError('RPI 0x71 not found in UDH')
+        rph_rhl_tar = remainder[:6] # RPH+RHL+TAR; not ciphered
+        res = self.SmsResponsePacket.parse(remainder)
+
+        if spi['por_shall_be_ciphered']:
+            # decrypt
+            ciphered_part = remainder[6:]
+            deciph = otak.crypt.decrypt(ciphered_part)
+            temp_data = rph_rhl_tar + deciph
+            res = self.SmsResponsePacket.parse(temp_data)
+            # remove specified number of padding bytes, if any
+            if res['pcntr'] != 0:
+                # this conditional is needed as python [:-0] renders an empty return!
+                res['secured_data'] = res['secured_data'][:-res['pcntr']]
+            remainder = temp_data
+
+        # is there a CC/RC present?
+        len_sig = res['rhl'] - 10
+        if spi['por_rc_cc_ds'] == 'no_rc_cc_ds':
+            if len_sig:
+                raise OtaCheckError('No RC/CC/DS requested, but len_sig=%u' % len_sig)
+        elif spi['por_rc_cc_ds'] == 'cc':
+            # verify signature
+            # UDH is part of CC/RC!
+            udh = data[:3]
+            # RPL, RHL, TAR, CNTR, PCNTR and STSare part of CC/RC
+            rpl_rhl_tar_cntr_pcntr_sts = remainder[:13]
+            # remove the CC/RC bytes
+            temp_data = udh + rpl_rhl_tar_cntr_pcntr_sts + remainder[13+len_sig:]
+            cc = otak.auth.check_sig(temp_data, res['cc_rc'])
+        # TODO: CRC
+        else:
+            raise OtaCheckError('Unknown por_rc_cc_ds: %s' % spi['por_rc_cc_ds'])
+
+        # TODO: ExpandedRemoteResponse according to TS 102 226 5.2.2
+        if res.response_status == 'por_ok':
+            dec = CompactRemoteResp.parse(res['secured_data'])
+        else:
+            dec = None
+        return (res, dec)

pySim/profile.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+
+""" pySim: tell old 2G SIMs apart from UICC
+"""
+
+#
+# (C) 2021 by Sysmocom s.f.m.c. GmbH
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from pySim.commands import SimCardCommands
+from pySim.filesystem import CardApplication, interpret_sw
+from pySim.utils import all_subclasses
+import abc
+import operator
+
+
+def _mf_select_test(scc: SimCardCommands, cla_byte: str, sel_ctrl: str) -> bool:
+    cla_byte_bak = scc.cla_byte
+    sel_ctrl_bak = scc.sel_ctrl
+    scc.reset_card()
+
+    scc.cla_byte = cla_byte
+    scc.sel_ctrl = sel_ctrl
+    rc = True
+    try:
+        scc.select_file('3f00')
+    except:
+        rc = False
+
+    scc.reset_card()
+    scc.cla_byte = cla_byte_bak
+    scc.sel_ctrl = sel_ctrl_bak
+    return rc
+
+
+def match_uicc(scc: SimCardCommands) -> bool:
+    """ Try to access MF via UICC APDUs (3GPP TS 102.221), if this works, the
+    card is considered a UICC card.
+    """
+    return _mf_select_test(scc, "00", "0004")
+
+
+def match_sim(scc: SimCardCommands) -> bool:
+    """ Try to access MF via 2G APDUs (3GPP TS 11.11), if this works, the card
+    is also a simcard. This will be the case for most UICC cards, but there may
+    also be plain UICC cards without 2G support as well.
+    """
+    return _mf_select_test(scc, "a0", "0000")
+
+
+class CardProfile:
+    """A Card Profile describes a card, it's filesystem hierarchy, an [initial] list of
+    applications as well as profile-specific SW and shell commands.  Every card has
+    one card profile, but there may be multiple applications within that profile."""
+
+    def __init__(self, name, **kw):
+        """
+        Args:
+                desc (str) : Description
+                files_in_mf : List of CardEF instances present in MF
+                applications : List of CardApplications present on card
+                sw : List of status word definitions
+                shell_cmdsets : List of cmd2 shell command sets of profile-specific commands
+                cla : class byte that should be used with cards of this profile
+                sel_ctrl : selection control bytes class byte that should be used with cards of this profile
+        """
+        self.name = name
+        self.desc = kw.get("desc", None)
+        self.files_in_mf = kw.get("files_in_mf", [])
+        self.sw = kw.get("sw", {})
+        self.applications = kw.get("applications", [])
+        self.shell_cmdsets = kw.get("shell_cmdsets", [])
+        self.cla = kw.get("cla", "00")
+        self.sel_ctrl = kw.get("sel_ctrl", "0004")
+
+    def __str__(self):
+        return self.name
+
+    def add_application(self, app: CardApplication):
+        """Add an application to a card profile.
+
+        Args:
+                app : CardApplication instance to be added to profile
+        """
+        self.applications.append(app)
+
+    def interpret_sw(self, sw: str):
+        """Interpret a given status word within the profile.
+
+        Args:
+                sw : Status word as string of 4 hex digits
+
+        Returns:
+                Tuple of two strings
+        """
+        return interpret_sw(self.sw, sw)
+
+    @staticmethod
+    def decode_select_response(data_hex: str) -> object:
+        """Decode the response to a SELECT command.
+
+        This is the fall-back method which doesn't perform any decoding. It mostly
+        exists so specific derived classes can overload it for actual decoding.
+        This method is implemented in the profile and is only used when application
+        specific decoding cannot be performed (no ADF is selected).
+
+        Args:
+                data_hex: Hex string of the select response
+        """
+        return data_hex
+
+    @staticmethod
+    @abc.abstractmethod
+    def match_with_card(scc: SimCardCommands) -> bool:
+        """Check if the specific profile matches the card. This method is a
+        placeholder that is overloaded by specific dirived classes. The method
+        actively probes the card to make sure the profile class matches the
+        physical card. This usually also means that the card is reset during
+        the process, so this method must not be called at random times. It may
+        only be called on startup.
+
+        Args:
+                scc: SimCardCommands class
+        Returns:
+                match = True, no match = False
+        """
+        return False
+
+    @staticmethod
+    def pick(scc: SimCardCommands):
+        profiles = list(all_subclasses(CardProfile))
+        profiles.sort(key=operator.attrgetter('ORDER'))
+
+        for p in profiles:
+            if p.match_with_card(scc):
+                return p()
+
+        return None

pySim/sms.py

@@ -0,0 +1,394 @@
+"""Code related to SMS Encoding/Decoding"""
+# simplistic SMS T-PDU code, as unfortunately nobody bothered to port the python smspdu
+# module to python3, and I gave up after >= 3 hours of trying and failing to do so
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import typing
+import abc
+from pprint import pprint as pp
+from construct import Int8ub, Byte, Bytes, Bit, Flag, BitsInteger, Flag
+from construct import Struct, Enum, Tell, BitStruct, this, Padding
+from construct import Prefixed, GreedyRange, GreedyBytes
+
+from pySim.construct import HexAdapter, BcdAdapter, TonNpi
+from pySim.utils import Hexstr, h2b, b2h
+
+from smpp.pdu import pdu_types
+
+BytesOrHex = typing.Union[Hexstr, bytes]
+
+class UserDataHeader:
+    # a single IE in the user data header
+    ie_c = Struct('iei'/Int8ub, 'length'/Int8ub, 'value'/Bytes(this.length))
+    # parser for the full UDH: Length octet followed by sequence of IEs
+    _construct = Struct('ies'/Prefixed(Int8ub, GreedyRange(ie_c)),
+                        'data'/GreedyBytes)
+
+    def __init__(self, ies=[]):
+        self.ies = ies
+
+    def __repr__(self) -> str:
+        return 'UDH(%r)' % self.ies
+
+    def has_ie(self, iei:int) -> bool:
+        for ie in self.ies:
+            if ie['iei'] == iei:
+                return True
+        return False
+
+    @classmethod
+    def fromBytes(cls, inb: BytesOrHex) -> typing.Tuple['UserDataHeader', bytes]:
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        res = cls._construct.parse(inb)
+        return cls(res['ies']), res['data']
+
+    def toBytes(self) -> bytes:
+        return self._construct.build({'ies':self.ies, 'data':b''})
+
+
+def smpp_dcs_is_8bit(dcs: pdu_types.DataCoding) -> bool:
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
+        return True
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED_COMMON):
+        return True
+    if dcs.scheme == pdu_types.DataCodingScheme.GSM_MESSAGE_CLASS and dcs.schemeData['msgCoding'] == pdu_types.DataCodingGsmMsgCoding.DATA_8BIT:
+        return True
+    else:
+        return False
+
+def ensure_smpp_is_8bit(dcs: pdu_types.DataCoding):
+    if not smpp_dcs_is_8bit(smpp_pdu.params['data_coding']):
+        raise ValueError('We only support 8bit coded SMS for now')
+
+class AddressField:
+    """Representation of an address field as used in SMS T-PDU."""
+    _construct = Struct('addr_len'/Int8ub,
+                        'type_of_addr'/TonNpi,
+                        'digits'/BcdAdapter(Bytes(this.addr_len//2 + this.addr_len%2)),
+                        'tell'/Tell)
+
+    def __init__(self, digits, ton='unknown', npi='unknown'):
+        self.ton = ton
+        self.npi = npi
+        self.digits = digits
+
+    def __str__(self):
+        return 'AddressField(TON=%s, NPI=%s, %s)' % (self.ton, self.npi, self.digits)
+
+    @classmethod
+    def fromBytes(cls, inb: BytesOrHex) -> typing.Tuple['AddressField', bytes]:
+        """Construct an AddressField instance from the binary T-PDU address format."""
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        res = cls._construct.parse(inb)
+        #pp(res)
+        #print("size: %s" % cls._construct.sizeof())
+        ton = res['type_of_addr']['type_of_number']
+        npi = res['type_of_addr']['numbering_plan_id']
+        # return resulting instance + remainder bytes
+        return cls(res['digits'][:res['addr_len']], ton, npi), inb[res['tell']:]
+
+    @classmethod
+    def fromSmpp(cls, addr, ton, npi) -> 'AddressField':
+        """Construct an AddressField from {source,dest}_addr_{,ton,npi} attributes of smpp.pdu."""
+        smpp_map_npi = {
+            'UNKNOWN': 'unknown',
+            'ISDN': 'isdn_e164',
+            'DATA': 'data_x121',
+            'TELEX': 'telex_f69',
+            'LAND_MOBILE': 'sc_specific6',
+            'NATIONAL': 'national',
+            'PRIVATE': 'private',
+            'ERMES': 'ermes',
+            }
+        smpp_map_ton = {
+            'UNKNOWN': 'unknown',
+            'INTERNATIONAL': 'international',
+            'NATIONAL': 'national',
+            'NETWORK_SPECIFIC': 'network_specific',
+            'SUBSCRIBER_NUMBER': 'short_code',
+            'ALPHANUMERIC': 'alphanumeric',
+            'ABBREVIATED': 'abbreviated',
+            }
+        # return the resulting instance
+        return cls(addr.decode('ascii'), smpp_map_ton[ton.name], smpp_map_npi[npi.name])
+
+
+    def toBytes(self) -> bytes:
+        """Encode the AddressField into the binary representation as used in T-PDU."""
+        num_digits = len(self.digits)
+        if num_digits % 2:
+            self.digits += 'f'
+        d = {
+            'addr_len': num_digits,
+            'type_of_addr': {
+                'ext': True,
+                'type_of_number': self.ton,
+                'numbering_plan_id': self.npi,
+                },
+            'digits': self.digits,
+            }
+        return self._construct.build(d)
+
+
+class SMS_TPDU(abc.ABC):
+    """Base class for a SMS T-PDU."""
+    def __init__(self, **kwargs):
+        self.tp_mti = kwargs.get('tp_mti', None)
+        self.tp_rp = kwargs.get('tp_rp', False)
+        self.tp_udhi = kwargs.get('tp_udhi', False)
+        self.tp_pid = kwargs.get('tp_pid', None)
+        self.tp_dcs = kwargs.get('tp_dcs', None)
+        self.tp_udl = kwargs.get('tp_udl', None)
+        self.tp_ud = kwargs.get('tp_ud', None)
+
+
+
+class SMS_DELIVER(SMS_TPDU):
+    """Representation of a SMS-DELIVER T-PDU."""
+    flags_construct = BitStruct('tp_rp'/Flag, 'tp_udhi'/Flag, 'tp_rp'/Flag, 'tp_sri'/Flag,
+                                Padding(1), 'tp_mms'/Flag, 'tp_mti'/BitsInteger(2))
+    def __init__(self, **kwargs):
+        kwargs['tp_mti'] = 0
+        super().__init__(**kwargs)
+        self.tp_lp = kwargs.get('tp_lp', False)
+        self.tp_mms = kwargs.get('tp_mms', False)
+        self.tp_oa = kwargs.get('tp_oa', None)
+        self.tp_scts = kwargs.get('tp_scts', None)
+        self.tp_sri = kwargs.get('tp_sri', False)
+
+    def __repr__(self):
+        return '%s(MTI=%s, MMS=%s, LP=%s, RP=%s, UDHI=%s, SRI=%s, OA=%s, PID=%2x, DCS=%x, SCTS=%s, UDL=%u, UD=%s)' % (self.__class__.__name__, self.tp_mti, self.tp_mms, self.tp_lp, self.tp_rp, self.tp_udhi, self.tp_sri, self.tp_oa, self.tp_pid, self.tp_dcs, self.tp_scts, self.tp_udl, self.tp_ud)
+
+    @classmethod
+    def fromBytes(cls, inb: BytesOrHex) -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from the binary encoded format as used in T-PDU."""
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        flags = inb[0]
+        d = SMS_DELIVER.flags_construct.parse(inb)
+        oa, remainder = AddressField.fromBytes(inb[1:])
+        d['tp_oa'] = oa
+        offset = 0
+        d['tp_pid'] = remainder[offset]
+        offset += 1
+        d['tp_dcs'] = remainder[offset]
+        offset += 1
+        # TODO: further decode
+        d['tp_scts'] = remainder[offset:offset+7]
+        offset += 7
+        d['tp_udl'] = remainder[offset]
+        offset += 1
+        d['tp_ud'] = remainder[offset:]
+        return cls(**d)
+
+    def toBytes(self) -> bytes:
+        """Encode a SMS_DELIVER instance to the binary encoded format as used in T-PDU."""
+        outb = bytearray()
+        d = {
+            'tp_mti': self.tp_mti, 'tp_mms': self.tp_mms, 'tp_lp': self.tp_lp,
+            'tp_rp': self.tp_rp, 'tp_udhi': self.tp_udhi, 'tp_sri': self.tp_sri,
+            }
+        flags = SMS_DELIVER.flags_construct.build(d)
+        outb.extend(flags)
+        outb.extend(self.tp_oa.toBytes())
+        outb.append(self.tp_pid)
+        outb.append(self.tp_dcs)
+        outb.extend(self.tp_scts)
+        outb.append(self.tp_udl)
+        outb.extend(self.tp_ud)
+
+        return outb
+
+    @classmethod
+    def fromSmpp(cls, smpp_pdu) -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from the deliver format used by smpp.pdu."""
+        if smpp_pdu.id == pdu_types.CommandId.submit_sm:
+            return cls.fromSmppSubmit(cls, smpp_pdu)
+        else:
+            raise ValueError('Unsupported SMPP commandId %s' % smpp_pdu.id)
+ 
+    @classmethod
+    def fromSmppSubmit(cls, smpp_pdu) -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from the submit format used by smpp.pdu."""
+        ensure_smpp_is_8bit(smpp_pdu.params['data_coding'])
+        tp_oa = AddressField.fromSmpp(smpp_pdu.params['source_addr'],
+                                      smpp_pdu.params['source_addr_ton'],
+                                      smpp_pdu.params['source_addr_npi'])
+        tp_ud = smpp_pdu.params['short_message']
+        d = {
+            'tp_lp': False,
+            'tp_mms': False,
+            'tp_oa': tp_oa,
+            'tp_scts': h2b('22705200000000'), # FIXME
+            'tp_sri': False,
+            'tp_rp': False,
+            'tp_udhi': pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET in smpp_pdu.params['esm_class'].gsmFeatures,
+            'tp_pid': smpp_pdu.params['protocol_id'],
+            'tp_dcs': 0xF6, # we only deal with binary SMS here
+            'tp_udl': len(tp_ud),
+            'tp_ud': tp_ud,
+            }
+        return cls(**d)
+
+
+
+class SMS_SUBMIT(SMS_TPDU):
+    """Representation of a SMS-DELIVER T-PDU."""
+    flags_construct = BitStruct('tp_srr'/Flag, 'tp_udhi'/Flag, 'tp_rp'/Flag,
+                                'tp_vpf'/Enum(BitsInteger(2), none=0, relative=2, enhanced=1, absolute=3),
+                                'tp_rd'/Flag, 'tp_mti'/BitsInteger(2))
+    def __init__(self, **kwargs):
+        kwargs['tp_mti'] = 1
+        super().__init__(**kwargs)
+        self.tp_rd = kwargs.get('tp_rd', False)
+        self.tp_vpf = kwargs.get('tp_vpf', 'none')
+        self.tp_srr = kwargs.get('tp_srr', False)
+        self.tp_mr = kwargs.get('tp_mr', None)
+        self.tp_da = kwargs.get('tp_da', None)
+        self.tp_vp = kwargs.get('tp_vp', None)
+
+    def __repr__(self):
+        return '%s(MTI=%s, RD=%s, VPF=%u, RP=%s, UDHI=%s, SRR=%s, DA=%s, PID=%2x, DCS=%x, VP=%s, UDL=%u, UD=%s)' % (self.__class__.__name__, self.tp_mti, self.tp_rd, self.tp_vpf, self.tp_rp, self.tp_udhi, self.tp_srr, self.tp_da, self.tp_pid, self.tp_dcs, self.tp_vp, self.tp_udl, self.tp_ud)
+
+    @classmethod
+    def fromBytes(cls, inb:BytesOrHex) -> 'SMS_SUBMIT':
+        """Construct a SMS_SUBMIT instance from the binary encoded format as used in T-PDU."""
+        offset = 0
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        d = SMS_SUBMIT.flags_construct.parse(inb)
+        offset += 1
+        d['tp_mr']= inb[offset]
+        offset += 1
+        da, remainder = AddressField.fromBytes(inb[2:])
+        d['tp_da'] = da
+
+        offset = 0
+        d['tp_pid'] = remainder[offset]
+        offset += 1
+        d['tp_dcs'] = remainder[offset]
+        offset += 1
+        if d['tp_vpf'] == 'none':
+            pass
+        elif d['tp_vpf'] == 'relative':
+            # TODO: further decode
+            d['tp_vp'] = remainder[offset:offset+1]
+            offset += 1
+        elif d['tp_vpf'] == 'enhanced':
+            # TODO: further decode
+            d['tp_vp'] = remainder[offset:offset+7]
+            offset += 7
+            pass
+        elif d['tp_vpf'] == 'absolute':
+            # TODO: further decode
+            d['tp_vp'] = remainder[offset:offset+7]
+            offset += 7
+            pass
+        else:
+            raise ValueError('Invalid VPF: %s' % d['tp_vpf'])
+        d['tp_udl'] = remainder[offset]
+        offset += 1
+        d['tp_ud'] = remainder[offset:]
+        return cls(**d)
+
+    def toBytes(self) -> bytes:
+        """Encode a SMS_SUBMIT instance to the binary encoded format as used in T-PDU."""
+        outb = bytearray()
+        d = {
+            'tp_mti': self.tp_mti, 'tp_rd': self.tp_rd, 'tp_vpf': self.tp_vpf,
+            'tp_rp': self.tp_rp, 'tp_udhi': self.tp_udhi, 'tp_srr': self.tp_srr,
+            }
+        flags = SMS_SUBMIT.flags_construct.build(d)
+        outb.extend(flags)
+        outb.append(self.tp_mr)
+        outb.extend(self.tp_da.toBytes())
+        outb.append(self.tp_pid)
+        outb.append(self.tp_dcs)
+        if self.tp_vpf != 'none':
+            outb.extend(self.tp_vp)
+        outb.append(self.tp_udl)
+        outb.extend(self.tp_ud)
+        return outb
+
+    @classmethod
+    def fromSmpp(cls, smpp_pdu) -> 'SMS_SUBMIT':
+        """Construct a SMS_DELIVER instance from the format used by smpp.pdu."""
+        if smpp_pdu.id == pdu_types.CommandId.submit_sm:
+            return cls.fromSmppSubmit(cls, smpp_pdu)
+        else:
+            raise ValueError('Unsupported SMPP commandId %s' % smpp_pdu.id)
+
+    @classmethod
+    def fromSmppSubmit(cls, smpp_pdu) -> 'SMS_SUBMIT':
+        """Construct a SMS_DELIVER instance from the submit format used by smpp.pdu."""
+        ensure_smpp_is_8bit(smpp_pdu.params['data_coding'])
+        tp_da = AddressField.fromSmpp(smpp_pdu.params['destination_addr'],
+                                      smpp_pdu.params['dest_addr_ton'],
+                                      smpp_pdu.params['dest_addr_npi'])
+        tp_ud = smpp_pdu.params['short_message']
+        #vp_smpp = smpp_pdu.params['validity_period']
+        #if not vp_smpp:
+        #    vpf = 'none'
+        d = {
+            'tp_rd': True if smpp_pdu.params['replace_if_present_flag'].name == 'REPLACE' else False,
+            'tp_vpf': None, # vpf,
+            'tp_rp': False, # related to ['registered_delivery'] ?
+            'tp_udhi': pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET in smpp_pdu.params['esm_class'].gsmFeatures,
+            'tp_srr': True if smpp_pdu.params['registered_delivery'] else False,
+            'tp_mr': 0, # FIXME: sm_default_msg_id ?
+            'tp_da': tp_da,
+            'tp_pid': smpp_pdu.params['protocol_id'],
+            'tp_dcs': 0xF6, # FIXME: we only deal with binary SMS here
+            'tp_vp': None, # FIXME: implement VPF conversion
+            'tp_udl': len(tp_ud),
+            'tp_ud': tp_ud,
+            }
+        return cls(**d)
+
+    def toSmpp(self) -> pdu_types.PDU:
+        """Translate a SMS_DELIVER instance to a smpp.pdu.pdu_types.SubmitSM instance."""
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT)
+        reg_del = pdu_types.RegisteredDelivery(pdu_types.RegisteredDeliveryReceipt.NO_SMSC_DELIVERY_RECEIPT_REQUESTED)
+        if self.tp_rp:
+            repl_if = pdu_types.ReplaceIfPresentFlag.REPLACE
+        else:
+            repl_if = pdu_types.ReplaceIfPresentFlag.DO_NOT_REPLACE
+        # we only deal with binary SMS here:
+        if self.tp_dcs != 0xF6:
+            raise ValueError('Unsupported DCS: We only support DCS=0xF6 for now')
+        dc = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT, pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        return pdu_types.SubmitSM(service_type='',
+                                  source_addr_ton=pdu_types.AddrTon.ALPHANUMERIC,
+                                  source_addr_npi=pdu_types.AddrNpi.UNKNOWN,
+                                  source_addr='simcard',
+                                  dest_addr_ton=FIXME(self.tp_da.ton),
+                                  dest_addr_npi=FIXME(self.tp_da.npi),
+                                  destination_addr=self.tp_da.digits,
+                                  esm_class=esm_class,
+                                  protocol_id=self.tp_pid,
+                                  priority_flag=pdu_types.PriorityFlag.LEVEL_0,
+                                  #schedule_delivery_time,
+                                  #validity_period,
+                                  registered_delivery=reg_del,
+                                  replace_if_present_flag=repl_if,
+                                  data_coding=dc,
+                                  #sm_default_msg_id,
+                                  short_message=self.tp_ud)

pySim/sysmocom_sja2.py

@@ -0,0 +1,302 @@
+# coding=utf-8
+"""Utilities / Functions related to sysmocom SJA2 cards
+
+(C) 2021 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from pytlv.TLV import *
+from struct import pack, unpack
+from pySim.utils import *
+from pySim.filesystem import *
+from pySim.ts_102_221 import CardProfileUICC
+from pySim.construct import *
+from construct import *
+import pySim
+
+key_type2str = {
+    0: 'kic',
+    1: 'kid',
+    2: 'kik',
+    3: 'any',
+}
+
+key_algo2str = {
+    0: 'des',
+    1: 'aes'
+}
+
+mac_length = {
+    0: 8,
+    1: 4
+}
+
+
+class EF_PIN(TransparentEF):
+    def __init__(self, fid, name):
+        super().__init__(fid, name=name, desc='%s PIN file' % name)
+
+    def _decode_bin(self, raw_bin_data):
+        u = unpack('!BBB8s', raw_bin_data[:11])
+        res = {'enabled': (True, False)[u[0] & 0x01],
+               'initialized': (True, False)[u[0] & 0x02],
+               'disable_able': (False, True)[u[0] & 0x10],
+               'unblock_able': (False, True)[u[0] & 0x20],
+               'change_able': (False, True)[u[0] & 0x40],
+               'valid': (False, True)[u[0] & 0x80],
+               'attempts_remaining': u[1],
+               'maximum_attempts': u[2],
+               'pin': u[3].hex(),
+               }
+        if len(raw_bin_data) == 21:
+            u2 = unpack('!BB8s', raw_bin_data[11:10])
+            res['attempts_remaining_puk'] = u2[0]
+            res['maximum_attempts_puk'] = u2[1]
+            res['puk'] = u2[2].hex()
+        return res
+
+
+class EF_MILENAGE_CFG(TransparentEF):
+    def __init__(self, fid='6f21', name='EF.MILENAGE_CFG', desc='Milenage connfiguration'):
+        super().__init__(fid, name=name, desc=desc)
+
+    def _decode_bin(self, raw_bin_data):
+        u = unpack('!BBBBB16s16s16s16s16s', raw_bin_data)
+        return {'r1': u[0], 'r2': u[1], 'r3': u[2], 'r4': u[3], 'r5': u[4],
+                'c1': u[5].hex(),
+                'c2': u[6].hex(),
+                'c3': u[7].hex(),
+                'c4': u[8].hex(),
+                'c5': u[9].hex(),
+                }
+
+
+class EF_0348_KEY(LinFixedEF):
+    def __init__(self, fid='6f22', name='EF.0348_KEY', desc='TS 03.48 OTA Keys'):
+        super().__init__(fid, name=name, desc=desc, rec_len=(27, 35))
+
+    def _decode_record_bin(self, raw_bin_data):
+        u = unpack('!BBB', raw_bin_data[0:3])
+        key_algo = (u[2] >> 6) & 1
+        key_length = ((u[2] >> 3) & 3) * 8
+        return {'sec_domain': u[0],
+                'key_set_version': u[1],
+                'key_type': key_type2str[u[2] & 3],
+                'key_length': key_length,
+                'algorithm': key_algo2str[key_algo],
+                'mac_length': mac_length[(u[2] >> 7)],
+                'key': raw_bin_data[3:key_length].hex()
+                }
+
+
+class EF_0348_COUNT(LinFixedEF):
+    def __init__(self, fid='6f23', name='EF.0348_COUNT', desc='TS 03.48 OTA Counters'):
+        super().__init__(fid, name=name, desc=desc, rec_len=(7, 7))
+
+    def _decode_record_bin(self, raw_bin_data):
+        u = unpack('!BB5s', raw_bin_data)
+        return {'sec_domain': u[0], 'key_set_version': u[1], 'counter': u[2]}
+
+
+class EF_SIM_AUTH_COUNTER(TransparentEF):
+    def __init__(self, fid='af24', name='EF.SIM_AUTH_COUNTER'):
+        super().__init__(fid, name=name, desc='Number of remaining RUN GSM ALGORITHM executions')
+        self._construct = Struct('num_run_gsm_algo_remain'/Int32ub)
+
+
+class EF_GP_COUNT(LinFixedEF):
+    def __init__(self, fid='6f26', name='EF.GP_COUNT', desc='GP SCP02 Counters'):
+        super().__init__(fid, name=name, desc=desc, rec_len=(5, 5))
+
+    def _decode_record_bin(self, raw_bin_data):
+        u = unpack('!BBHB', raw_bin_data)
+        return {'sec_domain': u[0], 'key_set_version': u[1], 'counter': u[2], 'rfu': u[3]}
+
+
+class EF_GP_DIV_DATA(LinFixedEF):
+    def __init__(self, fid='6f27', name='EF.GP_DIV_DATA', desc='GP SCP02 key diversification data'):
+        super().__init__(fid, name=name, desc=desc, rec_len=(12, 12))
+
+    def _decode_record_bin(self, raw_bin_data):
+        u = unpack('!BB8s', raw_bin_data)
+        return {'sec_domain': u[0], 'key_set_version': u[1], 'key_div_data': u[2].hex()}
+
+
+class EF_SIM_AUTH_KEY(TransparentEF):
+    def __init__(self, fid='6f20', name='EF.SIM_AUTH_KEY'):
+        super().__init__(fid, name=name, desc='USIM authentication key')
+        CfgByte = BitStruct(Padding(2),
+                            'use_sres_deriv_func_2'/Bit,
+                            'use_opc_instead_of_op'/Bit,
+                            'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
+        self._construct = Struct('cfg'/CfgByte,
+                                 'key'/HexAdapter(Bytes(16)),
+                                 'op'/ If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
+                                 'opc' /
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
+                                 )
+
+class EF_HTTPS_CFG(TransparentEF):
+    def __init__(self, fid='6f2a', name='EF.HTTPS_CFG'):
+        super().__init__(fid, name=name, desc='HTTPS configuration')
+
+class EF_HTTPS_KEYS(TransparentEF):
+    KeyRecord = Struct('security_domain'/Int8ub,
+                       'key_type'/Enum(Int8ub, des=0x80, psk=0x85, aes=0x88),
+                       'key_version'/Int8ub,
+                       'key_id'/Int8ub,
+                       'key_length'/Int8ub,
+                       'key'/HexAdapter(Bytes(this.key_length)))
+    def __init__(self, fid='6f2b', name='EF.HTTPS_KEYS'):
+        super().__init__(fid, name=name, desc='HTTPS PSK and DEK keys')
+        self._construct = GreedyRange(self.KeyRecord)
+
+class EF_HTTPS_POLL(TransparentEF):
+    TimeUnit = Enum(Int8ub, seconds=0, minutes=1, hours=2, days=3, ten_days=4)
+    def __init__(self, fid='6f2c', name='EF.HTTPS_POLL'):
+        super().__init__(fid, name=name, desc='HTTPS polling interval')
+        self._construct = Struct(Const(b'\x82'), 'time_unit'/self.TimeUnit, 'value'/Int8ub,
+                                 'adm_session_triggering_tlv'/HexAdapter(GreedyBytes))
+
+
+class DF_SYSTEM(CardDF):
+    def __init__(self):
+        super().__init__(fid='a515', name='DF.SYSTEM', desc='CardOS specifics')
+        files = [
+            EF_PIN('6f01', 'EF.CHV1'),
+            EF_PIN('6f81', 'EF.CHV2'),
+            EF_PIN('6f0a', 'EF.ADM1'),
+            EF_PIN('6f0b', 'EF.ADM2'),
+            EF_PIN('6f0c', 'EF.ADM3'),
+            EF_PIN('6f0d', 'EF.ADM4'),
+            EF_MILENAGE_CFG(),
+            EF_0348_KEY(),
+            EF_SIM_AUTH_COUNTER(),
+            EF_SIM_AUTH_KEY(),
+            EF_0348_COUNT(),
+            EF_GP_COUNT(),
+            EF_GP_DIV_DATA(),
+            EF_HTTPS_CFG(),
+            EF_HTTPS_KEYS(),
+            EF_HTTPS_POLL(),
+        ]
+        self.add_files(files)
+
+    def decode_select_response(self, resp_hex):
+        return pySim.ts_102_221.CardProfileUICC.decode_select_response(resp_hex)
+
+
+class EF_USIM_SQN(TransparentEF):
+    def __init__(self, fid='af30', name='EF.USIM_SQN'):
+        super().__init__(fid, name=name, desc='SQN parameters for AKA')
+        Flag1 = BitStruct('skip_next_sqn_check'/Bit, 'delta_max_check'/Bit,
+                          'age_limit_check'/Bit, 'sqn_check'/Bit,
+                          'ind_len'/BitsInteger(4))
+        Flag2 = BitStruct('rfu'/BitsRFU(5), 'dont_clear_amf_for_macs'/Bit,
+                          'aus_concealed'/Bit, 'autn_concealed'/Bit)
+        self._construct = Struct('flag1'/Flag1, 'flag2'/Flag2,
+                                 'delta_max' /
+                                 BytesInteger(6), 'age_limit'/BytesInteger(6),
+                                 'freshness'/GreedyRange(BytesInteger(6)))
+
+
+class EF_USIM_AUTH_KEY(TransparentEF):
+    def __init__(self, fid='af20', name='EF.USIM_AUTH_KEY'):
+        super().__init__(fid, name=name, desc='USIM authentication key')
+        CfgByte = BitStruct(Padding(1), 'only_4bytes_res_in_3g'/Bit,
+                            'use_sres_deriv_func_2_in_3g'/Bit,
+                            'use_opc_instead_of_op'/Bit,
+                            'algorithm'/Enum(Nibble, milenage=4, sha1_aka=5, xor=15))
+        self._construct = Struct('cfg'/CfgByte,
+                                 'key'/HexAdapter(Bytes(16)),
+                                 'op' /
+                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
+                                 'opc' /
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
+                                 )
+
+
+class EF_USIM_AUTH_KEY_2G(TransparentEF):
+    def __init__(self, fid='af22', name='EF.USIM_AUTH_KEY_2G'):
+        super().__init__(fid, name=name, desc='USIM authentication key in 2G context')
+        CfgByte = BitStruct(Padding(1), 'only_4bytes_res_in_3g'/Bit,
+                            'use_sres_deriv_func_2_in_3g'/Bit,
+                            'use_opc_instead_of_op'/Bit,
+                            'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
+        self._construct = Struct('cfg'/CfgByte,
+                                 'key'/HexAdapter(Bytes(16)),
+                                 'op' /
+                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
+                                 'opc' /
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
+                                 )
+
+
+class EF_GBA_SK(TransparentEF):
+    def __init__(self, fid='af31', name='EF.GBA_SK'):
+        super().__init__(fid, name=name, desc='Secret key for GBA key derivation')
+        self._construct = GreedyBytes
+
+
+class EF_GBA_REC_LIST(TransparentEF):
+    def __init__(self, fid='af32', name='EF.GBA_REC_LIST'):
+        super().__init__(fid, name=name, desc='Secret key for GBA key derivation')
+        # integers representing record numbers in EF-GBANL
+        self._construct = GreedyRange(Int8ub)
+
+
+class EF_GBA_INT_KEY(LinFixedEF):
+    def __init__(self, fid='af33', name='EF.GBA_INT_KEY'):
+        super().__init__(fid, name=name,
+                         desc='Secret key for GBA key derivation', rec_len=(32, 32))
+        self._construct = GreedyBytes
+
+
+class SysmocomSJA2(CardModel):
+    _atrs = ["3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 30 34 05 4B A9",
+             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 31 33 02 51 B2",
+             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 52 75 31 04 51 D5"]
+
+    @classmethod
+    def add_files(cls, rs: RuntimeState):
+        """Add sysmocom SJA2 specific files to given RuntimeState."""
+        rs.mf.add_file(DF_SYSTEM())
+        # optional USIM application
+        if 'a0000000871002' in rs.mf.applications:
+            usim_adf = rs.mf.applications['a0000000871002']
+            files_adf_usim = [
+                EF_USIM_AUTH_KEY(),
+                EF_USIM_AUTH_KEY_2G(),
+                EF_GBA_SK(),
+                EF_GBA_REC_LIST(),
+                EF_GBA_INT_KEY(),
+                EF_USIM_SQN(),
+            ]
+            usim_adf.add_files(files_adf_usim)
+        # optional ISIM application
+        if 'a0000000871004' in rs.mf.applications:
+            isim_adf = rs.mf.applications['a0000000871004']
+            files_adf_isim = [
+                EF_USIM_AUTH_KEY(name='EF.ISIM_AUTH_KEY'),
+                EF_USIM_AUTH_KEY_2G(name='EF.ISIM_AUTH_KEY_2G'),
+                EF_USIM_SQN(name='EF.ISIM_SQN'),
+            ]
+            isim_adf.add_files(files_adf_isim)

pySim/tlv.py

@@ -0,0 +1,439 @@
+"""object-oriented TLV parser/encoder library."""
+
+# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from typing import Optional, List, Dict, Any, Tuple
+from bidict import bidict
+from construct import *
+
+from pySim.utils import bertlv_encode_len, bertlv_parse_len, bertlv_encode_tag, bertlv_parse_tag
+from pySim.utils import comprehensiontlv_encode_tag, comprehensiontlv_parse_tag
+from pySim.utils import bertlv_parse_one, comprehensiontlv_parse_one
+from pySim.utils import bertlv_parse_tag_raw, comprehensiontlv_parse_tag_raw
+
+from pySim.construct import parse_construct, LV, HexAdapter, BcdAdapter, BitsRFU, GsmStringAdapter
+from pySim.exceptions import *
+
+import inspect
+import abc
+import re
+
+def camel_to_snake(name):
+    name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower()
+
+class TlvMeta(abc.ABCMeta):
+    """Metaclass which we use to set some class variables at the time of defining a subclass.
+    This allows us to create subclasses for each TLV/IE type, where the class represents fixed
+    parameters like the tag/type and instances of it represent the actual TLV data."""
+    def __new__(metacls, name, bases, namespace, **kwargs):
+        #print("TlvMeta_new_(metacls=%s, name=%s, bases=%s, namespace=%s, kwargs=%s)" % (metacls, name, bases, namespace, kwargs))
+        x = super().__new__(metacls, name, bases, namespace)
+        # this becomes a _class_ variable, not an instance variable
+        x.tag = namespace.get('tag', kwargs.get('tag', None))
+        x.desc = namespace.get('desc', kwargs.get('desc', None))
+        nested = namespace.get('nested', kwargs.get('nested', None))
+        if nested is None or inspect.isclass(nested) and issubclass(nested, TLV_IE_Collection):
+            # caller has specified TLV_IE_Collection sub-class, we can directly reference it
+            x.nested_collection_cls = nested
+        else:
+            # caller passed list of other TLV classes that might possibly appear within us,
+            # build a dynamically-created TLV_IE_Collection sub-class and reference it
+            name = 'auto_collection_%s' % (name)
+            cls = type(name, (TLV_IE_Collection,), {'nested': nested})
+            x.nested_collection_cls = cls
+        return x
+
+
+class TlvCollectionMeta(abc.ABCMeta):
+    """Metaclass which we use to set some class variables at the time of defining a subclass.
+    This allows us to create subclasses for each Collection type, where the class represents fixed
+    parameters like the nested IE classes and instances of it represent the actual TLV data."""
+    def __new__(metacls, name, bases, namespace, **kwargs):
+        #print("TlvCollectionMeta_new_(metacls=%s, name=%s, bases=%s, namespace=%s, kwargs=%s)" % (metacls, name, bases, namespace, kwargs))
+        x = super().__new__(metacls, name, bases, namespace)
+        # this becomes a _class_ variable, not an instance variable
+        x.possible_nested = namespace.get('nested', kwargs.get('nested', None))
+        return x
+
+
+class Transcodable(abc.ABC):
+    _construct = None
+    """Base class for something that can be encoded + encoded.  Decoding and Encoding happens either
+     * via a 'construct' object stored in a derived class' _construct variable, or
+     * via a 'construct' object stored in an instance _construct variable, or
+     * via a derived class' _{to,from}_bytes() methods."""
+
+    def __init__(self):
+        self.encoded = None
+        self.decoded = None
+        self._construct = None
+
+    def to_bytes(self) -> bytes:
+        """Convert from internal representation to binary bytes.  Store the binary result
+        in the internal state and return it."""
+        if not self.decoded:
+            do = b''
+        elif self._construct:
+            do = self._construct.build(self.decoded, total_len=None)
+        elif self.__class__._construct:
+            do = self.__class__._construct.build(self.decoded, total_len=None)
+        else:
+            do = self._to_bytes()
+        self.encoded = do
+        return do
+
+    # not an abstractmethod, as it is only required if no _construct exists
+    def _to_bytes(self):
+        raise NotImplementedError('%s._to_bytes' % type(self).__name__)
+
+    def from_bytes(self, do: bytes):
+        """Convert from binary bytes to internal representation. Store the decoded result
+        in the internal state and return it."""
+        self.encoded = do
+        if self.encoded == b'':
+            self.decoded = None
+        elif self._construct:
+            self.decoded = parse_construct(self._construct, do)
+        elif self.__class__._construct:
+            self.decoded = parse_construct(self.__class__._construct, do)
+        else:
+            self.decoded = self._from_bytes(do)
+        return self.decoded
+
+    # not an abstractmethod, as it is only required if no _construct exists
+    def _from_bytes(self, do: bytes):
+        raise NotImplementedError('%s._from_bytes' % type(self).__name__)
+
+
+class IE(Transcodable, metaclass=TlvMeta):
+    # we specify the metaclass so any downstream subclasses will automatically use it
+    """Base class for various Information Elements. We understand the notion of a hierarchy
+    of IEs on top of the Transcodable class."""
+    # this is overridden by the TlvMeta metaclass, if it is used to create subclasses
+    nested_collection_cls = None
+    tag = None
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.nested_collection = None
+        if self.nested_collection_cls:
+            self.nested_collection = self.nested_collection_cls()
+        # if we are a constructed IE, [ordered] list of actual child-IE instances
+        self.children = kwargs.get('children', [])
+        self.decoded = kwargs.get('decoded', None)
+
+    def __repr__(self):
+        """Return a string representing the [nested] IE data (for print)."""
+        if len(self.children):
+            member_strs = [repr(x) for x in self.children]
+            return '%s(%s)' % (type(self).__name__, ','.join(member_strs))
+        else:
+            return '%s(%s)' % (type(self).__name__, self.decoded)
+
+    def to_dict(self):
+        """Return a JSON-serializable dict representing the [nested] IE data."""
+        if len(self.children):
+            v = [x.to_dict() for x in self.children]
+        else:
+            v = self.decoded
+        return {camel_to_snake(type(self).__name__): v}
+
+    def from_dict(self, decoded: dict):
+        """Set the IE internal decoded representation to data from the argument.
+        If this is a nested IE, the child IE instance list is re-created."""
+        if self.nested_collection:
+            self.children = self.nested_collection.from_dict(decoded)
+        else:
+            self.children = []
+            self.decoded = decoded
+
+    def is_constructed(self):
+        """Is this IE constructed by further nested IEs?"""
+        if len(self.children):
+            return True
+        else:
+            return False
+
+    @abc.abstractmethod
+    def to_ie(self) -> bytes:
+        """Convert the internal representation to entire IE including IE header."""
+
+    def to_bytes(self) -> bytes:
+        """Convert the internal representation _of the value part_ to binary bytes."""
+        if self.is_constructed():
+            # concatenate the encoded IE of all children to form the value part
+            out = b''
+            for c in self.children:
+                out += c.to_ie()
+            return out
+        else:
+            return super().to_bytes()
+
+    def from_bytes(self, do: bytes):
+        """Parse _the value part_ from binary bytes to internal representation."""
+        if self.nested_collection:
+            self.children = self.nested_collection.from_bytes(do)
+        else:
+            self.children = []
+            return super().from_bytes(do)
+
+
+class TLV_IE(IE):
+    """Abstract base class for various TLV type Information Elements."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def _compute_tag(self) -> int:
+        """Compute the tag (sometimes the tag encodes part of the value)."""
+        return self.tag
+
+    @classmethod
+    @abc.abstractmethod
+    def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]:
+        """Obtain the raw TAG at the start of the bytes provided by the user."""
+
+    @classmethod
+    @abc.abstractmethod
+    def _parse_len(cls, do: bytes) -> Tuple[int, bytes]:
+        """Obtain the length encoded at the start of the bytes provided by the user."""
+
+    @abc.abstractmethod
+    def _encode_tag(self) -> bytes:
+        """Encode the tag part. Must be provided by derived (TLV format specific) class."""
+
+    @abc.abstractmethod
+    def _encode_len(self, val: bytes) -> bytes:
+        """Encode the length part assuming a certain binary value. Must be provided by
+        derived (TLV format specific) class."""
+
+    def to_ie(self):
+        return self.to_tlv()
+
+    def to_tlv(self):
+        """Convert the internal representation to binary TLV bytes."""
+        val = self.to_bytes()
+        return self._encode_tag() + self._encode_len(val) + val
+
+    def from_tlv(self, do: bytes):
+        if len(do) == 0:
+            return {}, b''
+        (rawtag, remainder) = self.__class__._parse_tag_raw(do)
+        if rawtag:
+            if rawtag != self._compute_tag():
+                raise ValueError("%s: Encountered tag %s doesn't match our supported tag %s" %
+                                 (self, rawtag, self.tag))
+            (length, remainder) = self.__class__._parse_len(remainder)
+            value = remainder[:length]
+            remainder = remainder[length:]
+        else:
+            value = do
+            remainder = b''
+        dec = self.from_bytes(value)
+        return dec, remainder
+
+
+class BER_TLV_IE(TLV_IE):
+    """TLV_IE formatted as ASN.1 BER described in ITU-T X.690 8.1.2."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @classmethod
+    def _decode_tag(cls, do: bytes) -> Tuple[dict, bytes]:
+        return bertlv_parse_tag(do)
+
+    @classmethod
+    def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]:
+        return bertlv_parse_tag_raw(do)
+
+    @classmethod
+    def _parse_len(cls, do: bytes) -> Tuple[int, bytes]:
+        return bertlv_parse_len(do)
+
+    def _encode_tag(self) -> bytes:
+        return bertlv_encode_tag(self._compute_tag())
+
+    def _encode_len(self, val: bytes) -> bytes:
+        return bertlv_encode_len(len(val))
+
+
+class COMPR_TLV_IE(TLV_IE):
+    """TLV_IE formated as COMPREHENSION-TLV as described in ETSI TS 101 220."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.comprehension = False
+
+    @classmethod
+    def _decode_tag(cls, do: bytes) -> Tuple[dict, bytes]:
+        return comprehensiontlv_parse_tag(do)
+
+    @classmethod
+    def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]:
+        return comprehensiontlv_parse_tag_raw(do)
+
+    @classmethod
+    def _parse_len(cls, do: bytes) -> Tuple[int, bytes]:
+        return bertlv_parse_len(do)
+
+    def _encode_tag(self) -> bytes:
+        return comprehensiontlv_encode_tag(self._compute_tag())
+
+    def _encode_len(self, val: bytes) -> bytes:
+        return bertlv_encode_len(len(val))
+
+
+class TLV_IE_Collection(metaclass=TlvCollectionMeta):
+    # we specify the metaclass so any downstream subclasses will automatically use it
+    """A TLV_IE_Collection consists of multiple TLV_IE classes identified by their tags.
+    A given encoded DO may contain any of them in any order, and may contain multiple instances
+    of each DO."""
+    # this is overridden by the TlvCollectionMeta metaclass, if it is used to create subclasses
+    possible_nested = []
+
+    def __init__(self, desc=None, **kwargs):
+        self.desc = desc
+        #print("possible_nested: ", self.possible_nested)
+        self.members = kwargs.get('nested', self.possible_nested)
+        self.members_by_tag = {}
+        self.members_by_name = {}
+        self.members_by_tag = {m.tag: m for m in self.members}
+        self.members_by_name = {m.__name__: m for m in self.members}
+        # if we are a constructed IE, [ordered] list of actual child-IE instances
+        self.children = kwargs.get('children', [])
+        self.encoded = None
+
+    def __str__(self):
+        member_strs = [str(x) for x in self.members]
+        return '%s(%s)' % (type(self).__name__, ','.join(member_strs))
+
+    def __repr__(self):
+        member_strs = [repr(x) for x in self.members]
+        return '%s(%s)' % (self.__class__, ','.join(member_strs))
+
+    def __add__(self, other):
+        """Extending TLV_IE_Collections with other TLV_IE_Collections or TLV_IEs."""
+        if isinstance(other, TLV_IE_Collection):
+            # adding one collection to another
+            members = self.members + other.members
+            return TLV_IE_Collection(self.desc, nested=members)
+        elif inspect.isclass(other) and issubclass(other, TLV_IE):
+            # adding a member to a collection
+            return TLV_IE_Collection(self.desc, nested=self.members + [other])
+        else:
+            raise TypeError
+
+    def from_bytes(self, binary: bytes) -> List[TLV_IE]:
+        """Create a list of TLV_IEs from the collection based on binary input data.
+        Args:
+            binary : binary bytes of encoded data
+        Returns:
+            list of instances of TLV_IE sub-classes containing parsed data
+        """
+        self.encoded = binary
+        # list of instances of TLV_IE collection member classes appearing in the data
+        res = []
+        remainder = binary
+        first = next(iter(self.members_by_tag.values()))
+        # iterate until no binary trailer is left
+        while len(remainder):
+            # obtain the tag at the start of the remainder
+            tag, r = first._parse_tag_raw(remainder)
+            if tag == None:
+                return res
+            if tag in self.members_by_tag:
+                cls = self.members_by_tag[tag]
+                # create an instance and parse accordingly
+                inst = cls()
+                dec, remainder = inst.from_tlv(remainder)
+                res.append(inst)
+            else:
+                # unknown tag; create the related class on-the-fly using the same base class
+                name = 'unknown_%s_%X' % (first.__base__.__name__, tag)
+                cls = type(name, (first.__base__,), {'tag': tag, 'possible_nested': [],
+                                                     'nested_collection_cls': None})
+                cls._from_bytes = lambda s, a: {'raw': a.hex()}
+                cls._to_bytes = lambda s: bytes.fromhex(s.decoded['raw'])
+                # create an instance and parse accordingly
+                inst = cls()
+                dec, remainder = inst.from_tlv(remainder)
+                res.append(inst)
+        self.children = res
+        return res
+
+    def from_dict(self, decoded: List[dict]) -> List[TLV_IE]:
+        """Create a list of TLV_IE instances from the collection based on an array
+        of dicts, where they key indicates the name of the TLV_IE subclass to use."""
+        # list of instances of TLV_IE collection member classes appearing in the data
+        res = []
+        for i in decoded:
+            for k in i.keys():
+                if k in self.members_by_name:
+                    cls = self.members_by_name[k]
+                    inst = cls()
+                    inst.from_dict(i[k])
+                    res.append(inst)
+                else:
+                    raise ValueError('%s: Unknown TLV Class %s in %s; expected %s' %
+                                     (self, i[0], decoded, self.members_by_name.keys()))
+        self.children = res
+        return res
+
+    def to_dict(self):
+        return [x.to_dict() for x in self.children]
+
+    def to_bytes(self):
+        out = b''
+        for c in self.children:
+            out += c.to_tlv()
+        return out
+
+    def from_tlv(self, do):
+        return self.from_bytes(do)
+
+    def to_tlv(self):
+        return self.to_bytes()
+
+
+def flatten_dict_lists(inp):
+    """hierarchically flatten each list-of-dicts into a single dict. This is useful to
+       make the output of hierarchical TLV decoder structures flatter and more easy to read."""
+    def are_all_elements_dict(l):
+        for e in l:
+            if not isinstance(e, dict):
+                return False
+        return True
+
+    if isinstance(inp, list):
+        if are_all_elements_dict(inp):
+            # flatten into one shared dict
+            newdict = {}
+            for e in inp:
+                key = list(e.keys())[0]
+                newdict[key] = e[key]
+            inp = newdict
+            # process result as any native dict
+            return {k:flatten_dict_lists(inp[k]) for k in inp.keys()}
+        else:
+            return [flatten_dict_lists(x) for x in inp]
+    elif isinstance(inp, dict):
+        return {k:flatten_dict_lists(inp[k]) for k in inp.keys()}
+    else:
+        return inp

pySim/transport/__init__.py

@@ -0,0 +1,319 @@
+# -*- coding: utf-8 -*-
+
+""" pySim: PCSC reader transport link base
+"""
+
+import abc
+import argparse
+from typing import Optional, Tuple
+
+from pySim.exceptions import *
+from pySim.construct import filter_dict
+from pySim.utils import sw_match, b2h, h2b, i2h, Hexstr
+from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
+
+#
+# Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
+# Copyright (C) 2021-2022 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class ApduTracer:
+    def trace_command(self, cmd):
+        pass
+
+    def trace_response(self, cmd, sw, resp):
+        pass
+
+class ProactiveHandler(abc.ABC):
+    """Abstract base class representing the interface of some code that handles
+    the proactive commands, as returned by the card in responses to the FETCH
+    command."""
+    def receive_fetch_raw(self, pcmd: ProactiveCommand, parsed: Hexstr):
+        # try to find a generic handler like handle_SendShortMessage
+        handle_name = 'handle_%s' % type(parsed).__name__
+        if hasattr(self, handle_name):
+            handler = getattr(self, handle_name)
+            return handler(pcmd.decoded)
+        # fall back to common handler
+        return self.receive_fetch(pcmd)
+
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        """Default handler for not otherwise handled proactive commands."""
+        raise NotImplementedError('No handler method for %s' % pcmd.decoded)
+
+
+
+class LinkBase(abc.ABC):
+    """Base class for link/transport to card."""
+
+    def __init__(self, sw_interpreter=None, apdu_tracer=None,
+                 proactive_handler: Optional[ProactiveHandler]=None):
+        self.sw_interpreter = sw_interpreter
+        self.apdu_tracer = apdu_tracer
+        self.proactive_handler = proactive_handler
+
+    @abc.abstractmethod
+    def _send_apdu_raw(self, pdu: str) -> Tuple[str, str]:
+        """Implementation specific method for sending the PDU."""
+
+    def set_sw_interpreter(self, interp):
+        """Set an (optional) status word interpreter."""
+        self.sw_interpreter = interp
+
+    @abc.abstractmethod
+    def wait_for_card(self, timeout: int = None, newcardonly: bool = False):
+        """Wait for a card and connect to it
+
+        Args:
+           timeout : Maximum wait time in seconds (None=no timeout)
+           newcardonly : Should we wait for a new card, or an already inserted one ?
+        """
+
+    @abc.abstractmethod
+    def connect(self):
+        """Connect to a card immediately
+        """
+
+    @abc.abstractmethod
+    def disconnect(self):
+        """Disconnect from card
+        """
+
+    @abc.abstractmethod
+    def reset_card(self):
+        """Resets the card (power down/up)
+        """
+
+    def send_apdu_raw(self, pdu: str):
+        """Sends an APDU with minimal processing
+
+        Args:
+           pdu : string of hexadecimal characters (ex. "A0A40000023F00")
+        Returns:
+           tuple(data, sw), where
+                        data : string (in hex) of returned data (ex. "074F4EFFFF")
+                        sw   : string (in hex) of status word (ex. "9000")
+        """
+        if self.apdu_tracer:
+            self.apdu_tracer.trace_command(pdu)
+        (data, sw) = self._send_apdu_raw(pdu)
+        if self.apdu_tracer:
+            self.apdu_tracer.trace_response(pdu, sw, data)
+        return (data, sw)
+
+    def send_apdu(self, pdu):
+        """Sends an APDU and auto fetch response data
+
+        Args:
+           pdu : string of hexadecimal characters (ex. "A0A40000023F00")
+        Returns:
+           tuple(data, sw), where
+                        data : string (in hex) of returned data (ex. "074F4EFFFF")
+                        sw   : string (in hex) of status word (ex. "9000")
+        """
+        data, sw = self.send_apdu_raw(pdu)
+
+        # When we have sent the first APDU, the SW may indicate that there are response bytes
+        # available. There are two SWs commonly used for this 9fxx (sim) and 61xx (usim), where
+        # xx is the number of response bytes available.
+        # See also:
+        if (sw is not None):
+            if ((sw[0:2] == '9f') or (sw[0:2] == '61')):
+                # SW1=9F: 3GPP TS 51.011 9.4.1, Responses to commands which are correctly executed
+                # SW1=61: ISO/IEC 7816-4, Table 5 — General meaning of the interindustry values of SW1-SW2
+                pdu_gr = pdu[0:2] + 'c00000' + sw[2:4]
+                data, sw = self.send_apdu_raw(pdu_gr)
+            if sw[0:2] == '6c':
+                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
+                pdu_gr = pdu[0:8] + sw[2:4]
+                data, sw = self.send_apdu_raw(pdu_gr)
+
+        return data, sw
+
+    def send_apdu_checksw(self, pdu, sw="9000"):
+        """Sends an APDU and check returned SW
+
+        Args:
+           pdu : string of hexadecimal characters (ex. "A0A40000023F00")
+           sw : string of 4 hexadecimal characters (ex. "9000"). The user may mask out certain
+                        digits using a '?' to add some ambiguity if needed.
+        Returns:
+                tuple(data, sw), where
+                        data : string (in hex) of returned data (ex. "074F4EFFFF")
+                        sw   : string (in hex) of status word (ex. "9000")
+        """
+        rv = self.send_apdu(pdu)
+        last_sw = rv[1]
+
+        while sw == '9000' and sw_match(last_sw, '91xx'):
+            # It *was* successful after all -- the extra pieces FETCH handled
+            # need not concern the caller.
+            rv = (rv[0], '9000')
+            # proactive sim as per TS 102 221 Setion 7.4.2
+            # TODO: Check SW manually to avoid recursing on the stack (provided this piece of code stays in this place)
+            fetch_rv = self.send_apdu_checksw('80120000' + last_sw[2:], sw)
+            # Setting this in case we later decide not to send a terminal
+            # response immediately unconditionally -- the card may still have
+            # something pending even though the last command was not processed
+            # yet.
+            last_sw = fetch_rv[1]
+            # parse the proactive command
+            pcmd = ProactiveCommand()
+            parsed = pcmd.from_tlv(h2b(fetch_rv[0]))
+            print("FETCH: %s (%s)" % (fetch_rv[0], type(parsed).__name__))
+            result = Result()
+            if self.proactive_handler:
+                # Extension point: If this does return a list of TLV objects,
+                # they could be appended after the Result; if the first is a
+                # Result, that cuold replace the one built here.
+                self.proactive_handler.receive_fetch_raw(pcmd, parsed)
+                result.from_dict({'general_result': 'performed_successfully', 'additional_information': ''})
+            else:
+                result.from_dict({'general_result': 'command_beyond_terminal_capability', 'additional_information': ''})
+
+            # Send response immediately, thus also flushing out any further
+            # proactive commands that the card already wants to send
+            #
+            # Structure as per TS 102 223 V4.4.0 Section 6.8
+
+            # The Command Details are echoed from the command that has been processed.
+            (command_details,) = [c for c in pcmd.decoded.children if isinstance(c, CommandDetails)]
+            # The Device Identities are fixed. (TS 102 223 V4.0.0 Section 6.8.2)
+            device_identities = DeviceIdentities()
+            device_identities.from_dict({'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'})
+
+            # Testing hint: The value of tail does not influence the behavior
+            # of an SJA2 that sent ans SMS, so this is implemented only
+            # following TS 102 223, and not fully tested.
+            tail = command_details.to_tlv() + device_identities.to_tlv() + result.to_tlv()
+            # Testing hint: In contrast to the above, this part is positively
+            # essential to get the SJA2 to provide the later parts of a
+            # multipart SMS in response to an OTA RFM command.
+            terminal_response = '80140000' + b2h(len(tail).to_bytes(1, 'big') + tail)
+
+            terminal_response_rv = self.send_apdu(terminal_response)
+            last_sw = terminal_response_rv[1]
+
+        if not sw_match(rv[1], sw):
+            raise SwMatchError(rv[1], sw.lower(), self.sw_interpreter)
+        return rv
+
+    def send_apdu_constr(self, cla, ins, p1, p2, cmd_constr, cmd_data, resp_constr):
+        """Build and sends an APDU using a 'construct' definition; parses response.
+
+        Args:
+                cla : string (in hex) ISO 7816 class byte
+                ins : string (in hex) ISO 7816 instruction byte
+                p1 : string (in hex) ISO 7116 Parameter 1 byte
+                p2 : string (in hex) ISO 7116 Parameter 2 byte
+                cmd_cosntr : defining how to generate binary APDU command data
+                cmd_data : command data passed to cmd_constr
+                resp_cosntr : defining how to decode  binary APDU response data
+        Returns:
+                Tuple of (decoded_data, sw)
+        """
+        cmd = cmd_constr.build(cmd_data) if cmd_data else ''
+        p3 = i2h([len(cmd)])
+        pdu = ''.join([cla, ins, p1, p2, p3, b2h(cmd)])
+        (data, sw) = self.send_apdu(pdu)
+        if data:
+            # filter the resulting dict to avoid '_io' members inside
+            rsp = filter_dict(resp_constr.parse(h2b(data)))
+        else:
+            rsp = None
+        return (rsp, sw)
+
+    def send_apdu_constr_checksw(self, cla, ins, p1, p2, cmd_constr, cmd_data, resp_constr,
+                                 sw_exp="9000"):
+        """Build and sends an APDU using a 'construct' definition; parses response.
+
+        Args:
+                cla : string (in hex) ISO 7816 class byte
+                ins : string (in hex) ISO 7816 instruction byte
+                p1 : string (in hex) ISO 7116 Parameter 1 byte
+                p2 : string (in hex) ISO 7116 Parameter 2 byte
+                cmd_cosntr : defining how to generate binary APDU command data
+                cmd_data : command data passed to cmd_constr
+                resp_cosntr : defining how to decode  binary APDU response data
+                exp_sw : string (in hex) of status word (ex. "9000")
+        Returns:
+                Tuple of (decoded_data, sw)
+        """
+        (rsp, sw) = self.send_apdu_constr(cla, ins,
+                                          p1, p2, cmd_constr, cmd_data, resp_constr)
+        if not sw_match(sw, sw_exp):
+            raise SwMatchError(sw, sw_exp.lower(), self.sw_interpreter)
+        return (rsp, sw)
+
+
+def argparse_add_reader_args(arg_parser):
+    """Add all reader related arguments to the given argparse.Argumentparser instance."""
+    serial_group = arg_parser.add_argument_group('Serial Reader')
+    serial_group.add_argument('-d', '--device', metavar='DEV', default='/dev/ttyUSB0',
+                              help='Serial Device for SIM access')
+    serial_group.add_argument('-b', '--baud', dest='baudrate', type=int, metavar='BAUD', default=9600,
+                              help='Baud rate used for SIM access')
+
+    pcsc_group = arg_parser.add_argument_group('PC/SC Reader')
+    pcsc_group.add_argument('-p', '--pcsc-device', type=int, dest='pcsc_dev', metavar='PCSC', default=None,
+                            help='PC/SC reader number to use for SIM access')
+
+    modem_group = arg_parser.add_argument_group('AT Command Modem Reader')
+    modem_group.add_argument('--modem-device', dest='modem_dev', metavar='DEV', default=None,
+                             help='Serial port of modem for Generic SIM Access (3GPP TS 27.007)')
+    modem_group.add_argument('--modem-baud', type=int, metavar='BAUD', default=115200,
+                             help='Baud rate used for modem port')
+
+    osmobb_group = arg_parser.add_argument_group('OsmocomBB Reader')
+    osmobb_group.add_argument('--osmocon', dest='osmocon_sock', metavar='PATH', default=None,
+                              help='Socket path for Calypso (e.g. Motorola C1XX) based reader (via OsmocomBB)')
+
+    return arg_parser
+
+
+def init_reader(opts, **kwargs) -> Optional[LinkBase]:
+    """
+    Init card reader driver
+    """
+    sl = None  # type : :Optional[LinkBase]
+    try:
+        if opts.pcsc_dev is not None:
+            print("Using PC/SC reader interface")
+            from pySim.transport.pcsc import PcscSimLink
+            sl = PcscSimLink(opts.pcsc_dev, **kwargs)
+        elif opts.osmocon_sock is not None:
+            print("Using Calypso-based (OsmocomBB) reader interface")
+            from pySim.transport.calypso import CalypsoSimLink
+            sl = CalypsoSimLink(sock_path=opts.osmocon_sock, **kwargs)
+        elif opts.modem_dev is not None:
+            print("Using modem for Generic SIM Access (3GPP TS 27.007)")
+            from pySim.transport.modem_atcmd import ModemATCommandLink
+            sl = ModemATCommandLink(
+                device=opts.modem_dev, baudrate=opts.modem_baud, **kwargs)
+        else:  # Serial reader is default
+            print("Using serial reader interface")
+            from pySim.transport.serial import SerialSimLink
+            sl = SerialSimLink(device=opts.device,
+                               baudrate=opts.baudrate, **kwargs)
+        return sl
+    except Exception as e:
+        if str(e):
+            print("Card reader initialization failed with exception:\n" + str(e))
+        else:
+            print(
+                "Card reader initialization failed with an exception of type:\n" + str(type(e)))
+        return None

pySim/transport/calypso.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2018 Vadim Yanitskiy <axilirator@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import select
+import struct
+import socket
+import os
+
+from pySim.transport import LinkBase
+from pySim.exceptions import *
+from pySim.utils import h2b, b2h
+
+
+class L1CTLMessage:
+
+    # Every (encoded) L1CTL message has the following structure:
+    #  - msg_length (2 bytes, net order)
+    #  - l1ctl_hdr (packed structure)
+    #    - msg_type
+    #    - flags
+    #    - padding (2 spare bytes)
+    #  - ... payload ...
+
+    def __init__(self, msg_type, flags=0x00):
+        # Init L1CTL message header
+        self.data = struct.pack("BBxx", msg_type, flags)
+
+    def gen_msg(self):
+        return struct.pack("!H", len(self.data)) + self.data
+
+
+class L1CTLMessageReset(L1CTLMessage):
+
+    # L1CTL message types
+    L1CTL_RESET_REQ = 0x0d
+    L1CTL_RESET_IND = 0x07
+    L1CTL_RESET_CONF = 0x0e
+
+    # Reset types
+    L1CTL_RES_T_BOOT = 0x00
+    L1CTL_RES_T_FULL = 0x01
+    L1CTL_RES_T_SCHED = 0x02
+
+    def __init__(self, type=L1CTL_RES_T_FULL):
+        super(L1CTLMessageReset, self).__init__(self.L1CTL_RESET_REQ)
+        self.data += struct.pack("Bxxx", type)
+
+
+class L1CTLMessageSIM(L1CTLMessage):
+
+    # SIM related message types
+    L1CTL_SIM_REQ = 0x16
+    L1CTL_SIM_CONF = 0x17
+
+    def __init__(self, pdu):
+        super(L1CTLMessageSIM, self).__init__(self.L1CTL_SIM_REQ)
+        self.data += pdu
+
+
+class CalypsoSimLink(LinkBase):
+    """Transport Link for Calypso based phones."""
+
+    def __init__(self, sock_path: str = "/tmp/osmocom_l2", **kwargs):
+        super().__init__(**kwargs)
+        # Make sure that a given socket path exists
+        if not os.path.exists(sock_path):
+            raise ReaderError(
+                "There is no such ('%s') UNIX socket" % sock_path)
+
+        print("Connecting to osmocon at '%s'..." % sock_path)
+
+        # Establish a client connection
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self.sock.connect(sock_path)
+
+    def __del__(self):
+        self.sock.close()
+
+    def wait_for_rsp(self, exp_len=128):
+        # Wait for incoming data (timeout is 3 seconds)
+        s, _, _ = select.select([self.sock], [], [], 3.0)
+        if not s:
+            raise ReaderError("Timeout waiting for card response")
+
+        # Receive expected amount of bytes from osmocon
+        rsp = self.sock.recv(exp_len)
+        return rsp
+
+    def reset_card(self):
+        # Request FULL reset
+        req_msg = L1CTLMessageReset()
+        self.sock.send(req_msg.gen_msg())
+
+        # Wait for confirmation
+        rsp = self.wait_for_rsp()
+        rsp_msg = struct.unpack_from("!HB", rsp)
+        if rsp_msg[1] != L1CTLMessageReset.L1CTL_RESET_CONF:
+            raise ReaderError("Failed to reset Calypso PHY")
+
+    def connect(self):
+        self.reset_card()
+
+    def disconnect(self):
+        pass  # Nothing to do really ...
+
+    def wait_for_card(self, timeout=None, newcardonly=False):
+        pass  # Nothing to do really ...
+
+    def _send_apdu_raw(self, pdu):
+
+        # Request FULL reset
+        req_msg = L1CTLMessageSIM(h2b(pdu))
+        self.sock.send(req_msg.gen_msg())
+
+        # Read message length first
+        rsp = self.wait_for_rsp(struct.calcsize("!H"))
+        msg_len = struct.unpack_from("!H", rsp)[0]
+        if msg_len < struct.calcsize("BBxx"):
+            raise ReaderError("Missing L1CTL header for L1CTL_SIM_CONF")
+
+        # Read the whole message then
+        rsp = self.sock.recv(msg_len)
+
+        # Verify L1CTL header
+        hdr = struct.unpack_from("BBxx", rsp)
+        if hdr[0] != L1CTLMessageSIM.L1CTL_SIM_CONF:
+            raise ReaderError("Unexpected L1CTL message received")
+
+        # Verify the payload length
+        offset = struct.calcsize("BBxx")
+        if len(rsp) <= offset:
+            raise ProtocolError("Empty response from SIM?!?")
+
+        # Omit L1CTL header
+        rsp = rsp[offset:]
+
+        # Unpack data and SW
+        data = rsp[:-2]
+        sw = rsp[-2:]
+
+        return b2h(data), b2h(sw)

pySim/transport/modem_atcmd.py

@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2020 Vadim Yanitskiy <axilirator@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging as log
+import serial
+import time
+import re
+
+from pySim.transport import LinkBase
+from pySim.exceptions import *
+
+# HACK: if somebody needs to debug this thing
+# log.root.setLevel(log.DEBUG)
+
+
+class ModemATCommandLink(LinkBase):
+    """Transport Link for 3GPP TS 27.007 compliant modems."""
+
+    def __init__(self, device: str = '/dev/ttyUSB0', baudrate: int = 115200, **kwargs):
+        super().__init__(**kwargs)
+        self._sl = serial.Serial(device, baudrate, timeout=5)
+        self._echo = False		# this will be auto-detected by _check_echo()
+        self._device = device
+        self._atr = None
+
+        # Check the AT interface
+        self._check_echo()
+
+        # Trigger initial reset
+        self.reset_card()
+
+    def __del__(self):
+        if hasattr(self, '_sl'):
+            self._sl.close()
+
+    def send_at_cmd(self, cmd, timeout=0.2, patience=0.002):
+        # Convert from string to bytes, if needed
+        bcmd = cmd if type(cmd) is bytes else cmd.encode()
+        bcmd += b'\r'
+
+        # Clean input buffer from previous/unexpected data
+        self._sl.reset_input_buffer()
+
+        # Send command to the modem
+        log.debug('Sending AT command: %s', cmd)
+        try:
+            wlen = self._sl.write(bcmd)
+            assert(wlen == len(bcmd))
+        except:
+            raise ReaderError('Failed to send AT command: %s' % cmd)
+
+        rsp = b''
+        its = 1
+        t_start = time.time()
+        while True:
+            rsp = rsp + self._sl.read(self._sl.in_waiting)
+            lines = rsp.split(b'\r\n')
+            if len(lines) >= 2:
+                res = lines[-2]
+                if res == b'OK':
+                    log.debug('Command finished with result: %s', res)
+                    break
+                if res == b'ERROR' or res.startswith(b'+CME ERROR:'):
+                    log.error('Command failed with result: %s', res)
+                    break
+
+            if time.time() - t_start >= timeout:
+                log.info('Command finished with timeout >= %ss', timeout)
+                break
+            time.sleep(patience)
+            its += 1
+        log.debug('Command took %0.6fs (%d cycles a %fs)',
+                  time.time() - t_start, its, patience)
+
+        if self._echo:
+            # Skip echo chars
+            rsp = rsp[wlen:]
+        rsp = rsp.strip()
+        rsp = rsp.split(b'\r\n\r\n')
+
+        log.debug('Got response from modem: %s', rsp)
+        return rsp
+
+    def _check_echo(self):
+        """Verify the correct response to 'AT' command
+        and detect if inputs are echoed by the device
+
+        Although echo of inputs can be enabled/disabled via
+        ATE1/ATE0, respectively, we rather detect the current
+        configuration of the modem without any change.
+        """
+        # Next command shall not strip the echo from the response
+        self._echo = False
+        result = self.send_at_cmd('AT')
+
+        # Verify the response
+        if len(result) > 0:
+            if result[-1] == b'OK':
+                self._echo = False
+                return
+            elif result[-1] == b'AT\r\r\nOK':
+                self._echo = True
+                return
+        raise ReaderError(
+            'Interface \'%s\' does not respond to \'AT\' command' % self._device)
+
+    def reset_card(self):
+        # Reset the modem, just to be sure
+        if self.send_at_cmd('ATZ') != [b'OK']:
+            raise ReaderError('Failed to reset the modem')
+
+        # Make sure that generic SIM access is supported
+        if self.send_at_cmd('AT+CSIM=?') != [b'OK']:
+            raise ReaderError('The modem does not seem to support SIM access')
+
+        log.info('Modem at \'%s\' is ready!' % self._device)
+
+    def connect(self):
+        pass  # Nothing to do really ...
+
+    def disconnect(self):
+        pass  # Nothing to do really ...
+
+    def wait_for_card(self, timeout=None, newcardonly=False):
+        pass  # Nothing to do really ...
+
+    def _send_apdu_raw(self, pdu):
+        # Make sure pdu has upper case hex digits [A-F]
+        pdu = pdu.upper()
+
+        # Prepare the command as described in 8.17
+        cmd = 'AT+CSIM=%d,\"%s\"' % (len(pdu), pdu)
+        log.debug('Sending command: %s',  cmd)
+
+        # Send AT+CSIM command to the modem
+        # TODO: also handle +CME ERROR: <err>
+        rsp = self.send_at_cmd(cmd)
+        if len(rsp) != 2 or rsp[-1] != b'OK':
+            raise ReaderError('APDU transfer failed: %s' % str(rsp))
+        rsp = rsp[0]  # Get rid of b'OK'
+
+        # Make sure that the response has format: b'+CSIM: %d,\"%s\"'
+        try:
+            result = re.match(b'\+CSIM: (\d+),\"([0-9A-F]+)\"', rsp)
+            (rsp_pdu_len, rsp_pdu) = result.groups()
+        except:
+            raise ReaderError('Failed to parse response from modem: %s' % rsp)
+
+        # TODO: make sure we have at least SW
+        data = rsp_pdu[:-4].decode().lower()
+        sw = rsp_pdu[-4:].decode().lower()
+        log.debug('Command response: %s, %s',  data, sw)
+        return data, sw

pySim/transport/pcsc.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
+# Copyright (C) 2010  Harald Welte <laforge@gnumonks.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from smartcard.CardConnection import CardConnection
+from smartcard.CardRequest import CardRequest
+from smartcard.Exceptions import NoCardException, CardRequestTimeoutException, CardConnectionException, CardConnectionException
+from smartcard.System import readers
+
+from pySim.exceptions import NoCardError, ProtocolError, ReaderError
+from pySim.transport import LinkBase
+from pySim.utils import h2i, i2h
+
+
+class PcscSimLink(LinkBase):
+    """ pySim: PCSC reader transport link."""
+
+    def __init__(self, reader_number: int = 0, **kwargs):
+        super().__init__(**kwargs)
+        r = readers()
+        if reader_number >= len(r):
+            raise ReaderError('No reader found for number %d' % reader_number)
+        self._reader = r[reader_number]
+        self._con = self._reader.createConnection()
+
+    def __del__(self):
+        try:
+            # FIXME: this causes multiple warnings in Python 3.5.3
+            self._con.disconnect()
+        except:
+            pass
+        return
+
+    def wait_for_card(self, timeout: int = None, newcardonly: bool = False):
+        cr = CardRequest(readers=[self._reader],
+                         timeout=timeout, newcardonly=newcardonly)
+        try:
+            cr.waitforcard()
+        except CardRequestTimeoutException:
+            raise NoCardError()
+        self.connect()
+
+    def connect(self):
+        try:
+            # To avoid leakage of resources, make sure the reader
+            # is disconnected
+            self.disconnect()
+
+            # Explicitly select T=0 communication protocol
+            self._con.connect(CardConnection.T0_protocol)
+        except CardConnectionException:
+            raise ProtocolError()
+        except NoCardException:
+            raise NoCardError()
+
+    def get_atr(self):
+        return self._con.getATR()
+
+    def disconnect(self):
+        self._con.disconnect()
+
+    def reset_card(self):
+        self.disconnect()
+        self.connect()
+        return 1
+
+    def _send_apdu_raw(self, pdu):
+
+        apdu = h2i(pdu)
+
+        data, sw1, sw2 = self._con.transmit(apdu)
+
+        sw = [sw1, sw2]
+
+        # Return value
+        return i2h(data), i2h(sw)

pySim/transport/serial.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import serial
+import time
+import os.path
+
+from pySim.exceptions import NoCardError, ProtocolError
+from pySim.transport import LinkBase
+from pySim.utils import h2b, b2h
+
+
+class SerialSimLink(LinkBase):
+    """ pySim: Transport Link for serial (RS232) based readers included with simcard"""
+
+    def __init__(self, device: str = '/dev/ttyUSB0', baudrate: int = 9600, rst: str = '-rts',
+                 debug: bool = False, **kwargs):
+        super().__init__(**kwargs)
+        if not os.path.exists(device):
+            raise ValueError("device file %s does not exist -- abort" % device)
+        self._sl = serial.Serial(
+            port=device,
+            parity=serial.PARITY_EVEN,
+            bytesize=serial.EIGHTBITS,
+            stopbits=serial.STOPBITS_TWO,
+            timeout=1,
+            xonxoff=0,
+            rtscts=0,
+            baudrate=baudrate,
+        )
+        self._rst_pin = rst
+        self._debug = debug
+        self._atr = None
+
+    def __del__(self):
+        if (hasattr(self, "_sl")):
+            self._sl.close()
+
+    def wait_for_card(self, timeout=None, newcardonly=False):
+        # Direct try
+        existing = False
+
+        try:
+            self.reset_card()
+            if not newcardonly:
+                return
+            else:
+                existing = True
+        except NoCardError:
+            pass
+
+        # Poll ...
+        mt = time.time() + timeout if timeout is not None else None
+        pe = 0
+
+        while (mt is None) or (time.time() < mt):
+            try:
+                time.sleep(0.5)
+                self.reset_card()
+                if not existing:
+                    return
+            except NoCardError:
+                existing = False
+            except ProtocolError:
+                if existing:
+                    existing = False
+                else:
+                    # Tolerate a couple of protocol error ... can happen if
+                    # we try when the card is 'half' inserted
+                    pe += 1
+                    if (pe > 2):
+                        raise
+
+        # Timed out ...
+        raise NoCardError()
+
+    def connect(self):
+        self.reset_card()
+
+    def get_atr(self):
+        return self._atr
+
+    def disconnect(self):
+        pass  # Nothing to do really ...
+
+    def reset_card(self):
+        rv = self._reset_card()
+        if rv == 0:
+            raise NoCardError()
+        elif rv < 0:
+            raise ProtocolError()
+
+    def _reset_card(self):
+        self._atr = None
+        rst_meth_map = {
+            'rts': self._sl.setRTS,
+            'dtr': self._sl.setDTR,
+        }
+        rst_val_map = {'+': 0, '-': 1}
+
+        try:
+            rst_meth = rst_meth_map[self._rst_pin[1:]]
+            rst_val = rst_val_map[self._rst_pin[0]]
+        except:
+            raise ValueError('Invalid reset pin %s' % self._rst_pin)
+
+        rst_meth(rst_val)
+        time.sleep(0.1)  # 100 ms
+        self._sl.flushInput()
+        rst_meth(rst_val ^ 1)
+
+        b = self._rx_byte()
+        if not b:
+            return 0
+        if ord(b) != 0x3b:
+            return -1
+        self._dbg_print("TS: 0x%x Direct convention" % ord(b))
+
+        while ord(b) == 0x3b:
+            b = self._rx_byte()
+
+        if not b:
+            return -1
+        t0 = ord(b)
+        self._dbg_print("T0: 0x%x" % t0)
+        self._atr = [0x3b, ord(b)]
+
+        for i in range(4):
+            if t0 & (0x10 << i):
+                b = self._rx_byte()
+                self._atr.append(ord(b))
+                self._dbg_print("T%si = %x" % (chr(ord('A')+i), ord(b)))
+
+        for i in range(0, t0 & 0xf):
+            b = self._rx_byte()
+            self._atr.append(ord(b))
+            self._dbg_print("Historical = %x" % ord(b))
+
+        while True:
+            x = self._rx_byte()
+            if not x:
+                break
+            self._atr.append(ord(x))
+            self._dbg_print("Extra: %x" % ord(x))
+
+        return 1
+
+    def _dbg_print(self, s):
+        if self._debug:
+            print(s)
+
+    def _tx_byte(self, b):
+        self._sl.write(b)
+        r = self._sl.read()
+        if r != b:  # TX and RX are tied, so we must clear the echo
+            raise ProtocolError("Bad echo value. Expected %02x, got %s)" % (
+                ord(b), '%02x' % ord(r) if r else '(nil)'))
+
+    def _tx_string(self, s):
+        """This is only safe if it's guaranteed the card won't send any data
+        during the time of tx of the string !!!"""
+        self._sl.write(s)
+        r = self._sl.read(len(s))
+        if r != s:  # TX and RX are tied, so we must clear the echo
+            raise ProtocolError(
+                "Bad echo value (Expected: %s, got %s)" % (b2h(s), b2h(r)))
+
+    def _rx_byte(self):
+        return self._sl.read()
+
+    def _send_apdu_raw(self, pdu):
+
+        pdu = h2b(pdu)
+        data_len = pdu[4]  # P3
+
+        # Send first CLASS,INS,P1,P2,P3
+        self._tx_string(pdu[0:5])
+
+        # Wait ack which can be
+        #  - INS: Command acked -> go ahead
+        #  - 0x60: NULL, just wait some more
+        #  - SW1: The card can apparently proceed ...
+        while True:
+            b = self._rx_byte()
+            if ord(b) == pdu[1]:
+                break
+            elif b != '\x60':
+                # Ok, it 'could' be SW1
+                sw1 = b
+                sw2 = self._rx_byte()
+                nil = self._rx_byte()
+                if (sw2 and not nil):
+                    return '', b2h(sw1+sw2)
+
+                raise ProtocolError()
+
+        # Send data (if any)
+        if len(pdu) > 5:
+            self._tx_string(pdu[5:])
+
+        # Receive data (including SW !)
+        #  length = [P3 - tx_data (=len(pdu)-len(hdr)) + 2 (SW1//2) ]
+        to_recv = data_len - len(pdu) + 5 + 2
+
+        data = bytes(0)
+        while (len(data) < to_recv):
+            b = self._rx_byte()
+            if (to_recv == 2) and (b == '\x60'):  # Ignore NIL if we have no RX data (hack ?)
+                continue
+            if not b:
+                break
+            data += b
+
+        # Split datafield from SW
+        if len(data) < 2:
+            return None, None
+        sw = data[-2:]
+        data = data[0:-2]
+
+        # Return value
+        return b2h(data), b2h(sw)

pySim/ts_102_221.py

@@ -0,0 +1,828 @@
+# coding=utf-8
+"""Utilities / Functions related to ETSI TS 102 221, the core UICC spec.
+
+(C) 2021 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from construct import *
+from construct import Optional as COptional
+from pySim.construct import *
+from pySim.utils import *
+from pySim.filesystem import *
+from pySim.tlv import *
+from bidict import bidict
+from pySim.profile import CardProfile
+from pySim.profile import match_uicc
+from pySim.profile import match_sim
+import pySim.iso7816_4 as iso7816_4
+
+# A UICC will usually also support 2G functionality. If this is the case, we
+# need to add DF_GSM and DF_TELECOM along with the UICC related files
+from pySim.ts_51_011 import DF_GSM, DF_TELECOM
+
+ts_102_22x_cmdset = CardCommandSet('TS 102 22x', [
+    # TS 102 221 Section 10.1.2 Table 10.5 "Coding of Instruction Byte"
+    CardCommand('SELECT',                   0xA4, ['0X', '4X', '6X']),
+    CardCommand('STATUS',                   0xF2, ['8X', 'CX', 'EX']),
+    CardCommand('READ BINARY',              0xB0, ['0X', '4X', '6X']),
+    CardCommand('UPDATE BINARY',            0xD6, ['0X', '4X', '6X']),
+    CardCommand('READ RECORD',              0xB2, ['0X', '4X', '6X']),
+    CardCommand('UPDATE RECORD',            0xDC, ['0X', '4X', '6X']),
+    CardCommand('SEARCH RECORD',            0xA2, ['0X', '4X', '6X']),
+    CardCommand('INCREASE',                 0x32, ['8X', 'CX', 'EX']),
+    CardCommand('RETRIEVE DATA',            0xCB, ['8X', 'CX', 'EX']),
+    CardCommand('SET DATA',                 0xDB, ['8X', 'CX', 'EX']),
+    CardCommand('VERIFY PIN',               0x20, ['0X', '4X', '6X']),
+    CardCommand('CHANGE PIN',               0x24, ['0X', '4X', '6X']),
+    CardCommand('DISABLE PIN',              0x26, ['0X', '4X', '6X']),
+    CardCommand('ENABLE PIN',               0x28, ['0X', '4X', '6X']),
+    CardCommand('UNBLOCK PIN',              0x2C, ['0X', '4X', '6X']),
+    CardCommand('DEACTIVATE FILE',          0x04, ['0X', '4X', '6X']),
+    CardCommand('ACTIVATE FILE',            0x44, ['0X', '4X', '6X']),
+    CardCommand('AUTHENTICATE',             0x88, ['0X', '4X', '6X']),
+    CardCommand('AUTHENTICATE',             0x89, ['0X', '4X', '6X']),
+    CardCommand('GET CHALLENGE',            0x84, ['0X', '4X', '6X']),
+    CardCommand('TERMINAL CAPABILITY',      0xAA, ['8X', 'CX', 'EX']),
+    CardCommand('TERMINAL PROFILE',         0x10, ['80']),
+    CardCommand('ENVELOPE',                 0xC2, ['80']),
+    CardCommand('FETCH',                    0x12, ['80']),
+    CardCommand('TERMINAL RESPONSE',        0x14, ['80']),
+    CardCommand('MANAGE CHANNEL',           0x70, ['0X', '4X', '6X']),
+    CardCommand('MANAGE SECURE CHANNEL',    0x73, ['0X', '4X', '6X']),
+    CardCommand('TRANSACT DATA',            0x75, ['0X', '4X', '6X']),
+    CardCommand('SUSPEND UICC',             0x76, ['80']),
+    CardCommand('GET IDENTITY',             0x78, ['8X', 'CX', 'EX']),
+    CardCommand('EXCHANGE CAPABILITIES',    0x7A, ['80']),
+    CardCommand('GET RESPONSE',             0xC0, ['0X', '4X', '6X']),
+    # TS 102 222 Section 6.1 Table 1 "Coding of the commands"
+    CardCommand('CREATE FILE',              0xE0, ['0X', '4X']),
+    CardCommand('DELETE FILE',              0xE4, ['0X', '4X']),
+    CardCommand('DEACTIVATE FILE',          0x04, ['0X', '4X']),
+    CardCommand('ACTIVATE FILE',            0x44, ['0X', '4X']),
+    CardCommand('TERMINATE DF',             0xE6, ['0X', '4X']),
+    CardCommand('TERMINATE EF',             0xE8, ['0X', '4X']),
+    CardCommand('TERMINATE CARD USAGE',     0xFE, ['0X', '4X']),
+    CardCommand('RESIZE FILE',              0xD4, ['8X', 'CX']),
+])
+
+# ETSI TS 102 221 11.1.1.4.2
+class FileSize(BER_TLV_IE, tag=0x80):
+    _construct = GreedyInteger(minlen=2)
+
+# ETSI TS 102 221 11.1.1.4.2
+class TotalFileSize(BER_TLV_IE, tag=0x81):
+    _construct = GreedyInteger(minlen=2)
+
+# ETSI TS 102 221 11.1.1.4.3
+class FileDescriptor(BER_TLV_IE, tag=0x82):
+    class BerTlvAdapter(Adapter):
+        def _parse(self, obj, context, path):
+            if obj == 0x39:
+                return 'ber_tlv'
+            raise ValidationError
+        def _build(self, obj, context, path):
+            if obj == 'ber_tlv':
+                return 0x39
+            raise ValidationError
+
+    FDB = Select(BitStruct(Const(0, Bit), 'shareable'/Flag, 'structure'/BerTlvAdapter(Const(0x39, BitsInteger(6)))),
+                 BitStruct(Const(0, Bit), 'shareable'/Flag, 'file_type'/Enum(BitsInteger(3), working_ef=0, internal_ef=1, df=7),
+                           'structure'/Enum(BitsInteger(3), no_info_given=0, transparent=1, linear_fixed=2, cyclic=6))
+                )
+    _construct = Struct('file_descriptor_byte'/FDB, Const(b'\x21'),
+                        'record_len'/COptional(Int16ub), 'num_of_rec'/COptional(Int8ub))
+
+# ETSI TS 102 221 11.1.1.4.4
+class FileIdentifier(BER_TLV_IE, tag=0x83):
+    _construct = HexAdapter(GreedyBytes)
+
+# ETSI TS 102 221 11.1.1.4.5
+class DfName(BER_TLV_IE, tag=0x84):
+    _construct = HexAdapter(GreedyBytes)
+
+# ETSI TS 102 221 11.1.1.4.6.1
+class UiccCharacteristics(BER_TLV_IE, tag=0x80):
+    _construct = GreedyBytes
+
+# ETSI TS 102 221 11.1.1.4.6.2
+class ApplicationPowerConsumption(BER_TLV_IE, tag=0x81):
+    _construct = Struct('voltage_class'/Int8ub,
+                        'power_consumption_ma'/Int8ub,
+                        'reference_freq_100k'/Int8ub)
+
+# ETSI TS 102 221 11.1.1.4.6.3
+class MinApplicationClockFrequency(BER_TLV_IE, tag=0x82):
+    _construct = Int8ub
+
+# ETSI TS 102 221 11.1.1.4.6.4
+class AvailableMemory(BER_TLV_IE, tag=0x83):
+    _construct = GreedyInteger()
+
+# ETSI TS 102 221 11.1.1.4.6.5
+class FileDetails(BER_TLV_IE, tag=0x84):
+    _construct = FlagsEnum(Byte, der_coding_only=1)
+
+# ETSI TS 102 221 11.1.1.4.6.6
+class ReservedFileSize(BER_TLV_IE, tag=0x85):
+    _construct = GreedyInteger()
+
+# ETSI TS 102 221 11.1.1.4.6.7
+class MaximumFileSize(BER_TLV_IE, tag=0x86):
+    _construct = GreedyInteger()
+
+# ETSI TS 102 221 11.1.1.4.6.8
+class SupportedFilesystemCommands(BER_TLV_IE, tag=0x87):
+    _construct = FlagsEnum(Byte, terminal_capability=1)
+
+# ETSI TS 102 221 11.1.1.4.6.9
+class SpecificUiccEnvironmentConditions(BER_TLV_IE, tag=0x88):
+    _construct = BitStruct('rfu'/BitsRFU(4),
+                           'high_humidity_supported'/Flag,
+                           'temperature_class'/Enum(BitsInteger(3), standard=0, class_A=1, class_B=2, class_C=3))
+
+# ETSI TS 102 221 11.1.1.4.6.10
+class Platform2PlatformCatSecuredApdu(BER_TLV_IE, tag=0x89):
+    _construct = GreedyBytes
+
+# sysmoISIM-SJA2 specific
+class ToolkitAccessConditions(BER_TLV_IE, tag=0xD2):
+    _construct = FlagsEnum(Byte, rfm_create=1, rfm_delete_terminate=2, other_applet_create=4,
+                           other_applet_delete_terminate=8)
+
+# ETSI TS 102 221 11.1.1.4.6.0
+class ProprietaryInformation(BER_TLV_IE, tag=0xA5,
+                             nested=[UiccCharacteristics, ApplicationPowerConsumption,
+                                     MinApplicationClockFrequency, AvailableMemory,
+                                     FileDetails, ReservedFileSize, MaximumFileSize,
+                                     SupportedFilesystemCommands, SpecificUiccEnvironmentConditions,
+                                     ToolkitAccessConditions]):
+    pass
+
+# ETSI TS 102 221 11.1.1.4.7.1
+class SecurityAttribCompact(BER_TLV_IE, tag=0x8c):
+    _construct = GreedyBytes
+
+# ETSI TS 102 221 11.1.1.4.7.2
+class SecurityAttribExpanded(BER_TLV_IE, tag=0xab):
+    _construct = GreedyBytes
+
+# ETSI TS 102 221 11.1.1.4.7.3
+class SecurityAttribReferenced(BER_TLV_IE, tag=0x8b):
+    # TODO: longer format with SEID
+    _construct = Struct('ef_arr_file_id'/HexAdapter(Bytes(2)), 'ef_arr_record_nr'/Int8ub)
+
+# ETSI TS 102 221 11.1.1.4.8
+class ShortFileIdentifier(BER_TLV_IE, tag=0x88):
+    # If the length of the TLV is 1, the SFI value is indicated in the 5 most significant bits (bits b8 to b4)
+    # of the TLV value field. In this case, bits b3 to b1 shall be set to 0
+    class Shift3RAdapter(Adapter):
+        def _decode(self, obj, context, path):
+            return int.from_bytes(obj, 'big') >> 3
+        def _encode(self, obj, context, path):
+            val = int(obj) << 3
+            return val.to_bytes(1, 'big')
+    _construct = COptional(Shift3RAdapter(Bytes(1)))
+
+# ETSI TS 102 221 11.1.1.4.9
+class LifeCycleStatusInteger(BER_TLV_IE, tag=0x8A):
+    def _from_bytes(self, do: bytes):
+        lcsi = int.from_bytes(do, 'big')
+        if lcsi == 0x00:
+            ret = 'no_information'
+        elif lcsi == 0x01:
+            ret = 'creation'
+        elif lcsi == 0x03:
+            ret = 'initialization'
+        elif lcsi & 0x05 == 0x05:
+            ret = 'operational_activated'
+        elif lcsi & 0x05 == 0x04:
+            ret = 'operational_deactivated'
+        elif lcsi & 0xc0 == 0xc0:
+            ret = 'termination'
+        else:
+            ret = lcsi
+        self.decoded = ret
+        return self.decoded
+    def _to_bytes(self):
+        if self.decoded == 'no_information':
+            return b'\x00'
+        elif self.decoded == 'creation':
+            return b'\x01'
+        elif self.decoded == 'initialization':
+            return b'\x03'
+        elif self.decoded == 'operational_activated':
+            return b'\x05'
+        elif self.decoded == 'operational_deactivated':
+            return b'\x04'
+        elif self.decoded == 'termination':
+            return b'\x0c'
+        elif isinstance(self.decoded, int):
+            return self.decoded.to_bytes(1, 'big')
+        else:
+            raise ValueError
+
+# ETSI TS 102 221 11.1.1.4.9
+class PS_DO(BER_TLV_IE, tag=0x90):
+    _construct = GreedyBytes
+class UsageQualifier_DO(BER_TLV_IE, tag=0x95):
+    _construct = GreedyBytes
+class KeyReference(BER_TLV_IE, tag=0x83):
+    _construct = Byte
+class PinStatusTemplate_DO(BER_TLV_IE, tag=0xC6, nested=[PS_DO, UsageQualifier_DO, KeyReference]):
+    pass
+
+class FcpTemplate(BER_TLV_IE, tag=0x62, nested=[FileSize, TotalFileSize, FileDescriptor, FileIdentifier,
+                                                DfName, ProprietaryInformation, SecurityAttribCompact,
+                                                SecurityAttribExpanded, SecurityAttribReferenced,
+                                                ShortFileIdentifier, LifeCycleStatusInteger,
+                                                PinStatusTemplate_DO]):
+    pass
+
+
+def tlv_key_replace(inmap, indata):
+    def newkey(inmap, key):
+        if key in inmap:
+            return inmap[key]
+        else:
+            return key
+    return {newkey(inmap, d[0]): d[1] for d in indata.items()}
+
+
+def tlv_val_interpret(inmap, indata):
+    def newval(inmap, key, val):
+        if key in inmap:
+            return inmap[key](val)
+        else:
+            return val
+    return {d[0]: newval(inmap, d[0], d[1]) for d in indata.items()}
+
+# ETSI TS 102 221 Section 9.2.7 + ISO7816-4 9.3.3/9.3.4
+class _AM_DO_DF(DataObject):
+    def __init__(self):
+        super().__init__('access_mode', 'Access Mode', tag=0x80)
+
+    def from_bytes(self, do: bytes):
+        res = []
+        if len(do) != 1:
+            raise ValueError("We only support single-byte AMF inside AM-DO")
+        amf = do[0]
+        # tables 17..29 and 41..44 of 7816-4
+        if amf & 0x80 == 0:
+            if amf & 0x40:
+                res.append('delete_file')
+            if amf & 0x20:
+                res.append('terminate_df')
+            if amf & 0x10:
+                res.append('activate_file')
+            if amf & 0x08:
+                res.append('deactivate_file')
+        if amf & 0x04:
+            res.append('create_file_df')
+        if amf & 0x02:
+            res.append('create_file_ef')
+        if amf & 0x01:
+            res.append('delete_file_child')
+        self.decoded = res
+
+    def to_bytes(self):
+        val = 0
+        if 'delete_file' in self.decoded:
+            val |= 0x40
+        if 'terminate_df' in self.decoded:
+            val |= 0x20
+        if 'activate_file' in self.decoded:
+            val |= 0x10
+        if 'deactivate_file' in self.decoded:
+            val |= 0x08
+        if 'create_file_df' in self.decoded:
+            val |= 0x04
+        if 'create_file_ef' in self.decoded:
+            val |= 0x02
+        if 'delete_file_child' in self.decoded:
+            val |= 0x01
+        return val.to_bytes(1, 'big')
+
+
+class _AM_DO_EF(DataObject):
+    """ISO7816-4 9.3.2 Table 18 + 9.3.3.1 Table 31"""
+
+    def __init__(self):
+        super().__init__('access_mode', 'Access Mode', tag=0x80)
+
+    def from_bytes(self, do: bytes):
+        res = []
+        if len(do) != 1:
+            raise ValueError("We only support single-byte AMF inside AM-DO")
+        amf = do[0]
+        # tables 17..29 and 41..44 of 7816-4
+        if amf & 0x80 == 0:
+            if amf & 0x40:
+                res.append('delete_file')
+            if amf & 0x20:
+                res.append('terminate_ef')
+            if amf & 0x10:
+                res.append('activate_file_or_record')
+            if amf & 0x08:
+                res.append('deactivate_file_or_record')
+        if amf & 0x04:
+            res.append('write_append')
+        if amf & 0x02:
+            res.append('update_erase')
+        if amf & 0x01:
+            res.append('read_search_compare')
+        self.decoded = res
+
+    def to_bytes(self):
+        val = 0
+        if 'delete_file' in self.decoded:
+            val |= 0x40
+        if 'terminate_ef' in self.decoded:
+            val |= 0x20
+        if 'activate_file_or_record' in self.decoded:
+            val |= 0x10
+        if 'deactivate_file_or_record' in self.decoded:
+            val |= 0x08
+        if 'write_append' in self.decoded:
+            val |= 0x04
+        if 'update_erase' in self.decoded:
+            val |= 0x02
+        if 'read_search_compare' in self.decoded:
+            val |= 0x01
+        return val.to_bytes(1, 'big')
+
+
+class _AM_DO_CHDR(DataObject):
+    """Command Header Access Mode DO according to ISO 7816-4 Table 32."""
+
+    def __init__(self, tag):
+        super().__init__('command_header', 'Command Header Description', tag=tag)
+
+    def from_bytes(self, do: bytes):
+        res = {}
+        i = 0
+        if self.tag & 0x08:
+            res['CLA'] = do[i]
+            i += 1
+        if self.tag & 0x04:
+            res['INS'] = do[i]
+            i += 1
+        if self.tag & 0x02:
+            res['P1'] = do[i]
+            i += 1
+        if self.tag & 0x01:
+            res['P2'] = do[i]
+            i += 1
+        self.decoded = res
+
+    def _compute_tag(self):
+        """Override to encode the tag, as it depends on the value."""
+        tag = 0x80
+        if 'CLA' in self.decoded:
+            tag |= 0x08
+        if 'INS' in self.decoded:
+            tag |= 0x04
+        if 'P1' in self.decoded:
+            tag |= 0x02
+        if 'P2' in self.decoded:
+            tag |= 0x01
+        return tag
+
+    def to_bytes(self):
+        res = bytearray()
+        if 'CLA' in self.decoded:
+            res.append(self.decoded['CLA'])
+        if 'INS' in self.decoded:
+            res.append(self.decoded['INS'])
+        if 'P1' in self.decoded:
+            res.append(self.decoded['P1'])
+        if 'P2' in self.decoded:
+            res.append(self.decoded['P2'])
+        return res
+
+
+AM_DO_CHDR = DataObjectChoice('am_do_chdr', members=[
+    _AM_DO_CHDR(0x81), _AM_DO_CHDR(0x82), _AM_DO_CHDR(0x83), _AM_DO_CHDR(0x84),
+    _AM_DO_CHDR(0x85), _AM_DO_CHDR(0x86), _AM_DO_CHDR(0x87), _AM_DO_CHDR(0x88),
+    _AM_DO_CHDR(0x89), _AM_DO_CHDR(0x8a), _AM_DO_CHDR(0x8b), _AM_DO_CHDR(0x8c),
+    _AM_DO_CHDR(0x8d), _AM_DO_CHDR(0x8e), _AM_DO_CHDR(0x8f)])
+
+AM_DO_DF = AM_DO_CHDR | _AM_DO_DF()
+AM_DO_EF = AM_DO_CHDR | _AM_DO_EF()
+
+
+# TS 102 221 Section 9.5.1 / Table 9.3
+pin_names = bidict({
+    0x01: 'PIN1',
+    0x02: 'PIN2',
+    0x03: 'PIN3',
+    0x04: 'PIN4',
+    0x05: 'PIN5',
+    0x06: 'PIN6',
+    0x07: 'PIN7',
+    0x08: 'PIN8',
+    0x0a: 'ADM1',
+    0x0b: 'ADM2',
+    0x0c: 'ADM3',
+    0x0d: 'ADM4',
+    0x0e: 'ADM5',
+
+    0x11: 'UNIVERSAL_PIN',
+    0x81: '2PIN1',
+    0x82: '2PIN2',
+    0x83: '2PIN3',
+    0x84: '2PIN4',
+    0x85: '2PIN5',
+    0x86: '2PIN6',
+    0x87: '2PIN7',
+    0x88: '2PIN8',
+    0x8a: 'ADM6',
+    0x8b: 'ADM7',
+    0x8c: 'ADM8',
+    0x8d: 'ADM9',
+    0x8e: 'ADM10',
+})
+
+
+class CRT_DO(DataObject):
+    """Control Reference Template as per TS 102 221 9.5.1"""
+
+    def __init__(self):
+        super().__init__('control_reference_template',
+                         'Control Reference Template', tag=0xA4)
+
+    def from_bytes(self, do: bytes):
+        """Decode a Control Reference Template DO."""
+        if len(do) != 6:
+            raise ValueError('Unsupported CRT DO length: %s', do)
+        if do[0] != 0x83 or do[1] != 0x01:
+            raise ValueError('Unsupported Key Ref Tag or Len in CRT DO %s', do)
+        if do[3:] != b'\x95\x01\x08':
+            raise ValueError(
+                'Unsupported Usage Qualifier Tag or Len in CRT DO %s', do)
+        self.encoded = do[0:6]
+        self.decoded = pin_names[do[2]]
+        return do[6:]
+
+    def to_bytes(self):
+        pin = pin_names.inverse[self.decoded]
+        return b'\x83\x01' + pin.to_bytes(1, 'big') + b'\x95\x01\x08'
+
+# ISO7816-4 9.3.3 Table 33
+class SecCondByte_DO(DataObject):
+    def __init__(self, tag=0x9d):
+        super().__init__('security_condition_byte', tag=tag)
+
+    def from_bytes(self, binary: bytes):
+        if len(binary) != 1:
+            raise ValueError
+        inb = binary[0]
+        if inb == 0:
+            cond = 'always'
+        if inb == 0xff:
+            cond = 'never'
+        res = []
+        if inb & 0x80:
+            cond = 'and'
+        else:
+            cond = 'or'
+        if inb & 0x40:
+            res.append('secure_messaging')
+        if inb & 0x20:
+            res.append('external_auth')
+        if inb & 0x10:
+            res.append('user_auth')
+        rd = {'mode': cond}
+        if len(res):
+            rd['conditions'] = res
+        self.decoded = rd
+
+    def to_bytes(self):
+        mode = self.decoded['mode']
+        if mode == 'always':
+            res = 0
+        elif mode == 'never':
+            res = 0xff
+        else:
+            res = 0
+            if mode == 'and':
+                res |= 0x80
+            elif mode == 'or':
+                pass
+            else:
+                raise ValueError('Unknown mode %s' % mode)
+            for c in self.decoded['conditions']:
+                if c == 'secure_messaging':
+                    res |= 0x40
+                elif c == 'external_auth':
+                    res |= 0x20
+                elif c == 'user_auth':
+                    res |= 0x10
+                else:
+                    raise ValueError('Unknown condition %s' % c)
+        return res.to_bytes(1, 'big')
+
+
+Always_DO = TL0_DataObject('always', 'Always', 0x90)
+Never_DO = TL0_DataObject('never', 'Never', 0x97)
+
+
+class Nested_DO(DataObject):
+    """A DO that nests another DO/Choice/Sequence"""
+
+    def __init__(self, name, tag, choice):
+        super().__init__(name, tag=tag)
+        self.children = choice
+
+    def from_bytes(self, binary: bytes) -> list:
+        remainder = binary
+        self.decoded = []
+        while remainder:
+            rc, remainder = self.children.decode(remainder)
+            self.decoded.append(rc)
+        return self.decoded
+
+    def to_bytes(self) -> bytes:
+        encoded = [self.children.encode(d) for d in self.decoded]
+        return b''.join(encoded)
+
+
+OR_Template = DataObjectChoice('or_template', 'OR-Template',
+                               members=[Always_DO, Never_DO, SecCondByte_DO(), SecCondByte_DO(0x9e), CRT_DO()])
+OR_DO = Nested_DO('or', 0xa0, OR_Template)
+AND_Template = DataObjectChoice('and_template', 'AND-Template',
+                                members=[Always_DO, Never_DO, SecCondByte_DO(), SecCondByte_DO(0x9e), CRT_DO()])
+AND_DO = Nested_DO('and', 0xa7, AND_Template)
+NOT_Template = DataObjectChoice('not_template', 'NOT-Template',
+                                members=[Always_DO, Never_DO, SecCondByte_DO(), SecCondByte_DO(0x9e), CRT_DO()])
+NOT_DO = Nested_DO('not', 0xaf, NOT_Template)
+SC_DO = DataObjectChoice('security_condition', 'Security Condition',
+                         members=[Always_DO, Never_DO, SecCondByte_DO(), SecCondByte_DO(0x9e), CRT_DO(),
+                                  OR_DO, AND_DO, NOT_DO])
+
+# TS 102 221 Section 13.1
+class EF_DIR(LinFixedEF):
+    class ApplicationLabel(BER_TLV_IE, tag=0x50):
+        # TODO: UCS-2 coding option as per Annex A of TS 102 221
+        _construct = GreedyString('ascii')
+
+    # see https://github.com/PyCQA/pylint/issues/5794
+    #pylint: disable=undefined-variable
+    class ApplicationTemplate(BER_TLV_IE, tag=0x61,
+                              nested=[iso7816_4.ApplicationId, ApplicationLabel, iso7816_4.FileReference,
+                                      iso7816_4.CommandApdu, iso7816_4.DiscretionaryData,
+                                      iso7816_4.DiscretionaryTemplate, iso7816_4.URL,
+                                      iso7816_4.ApplicationRelatedDOSet]):
+        pass
+
+    def __init__(self, fid='2f00', sfid=0x1e, name='EF.DIR', desc='Application Directory'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(5, 54))
+        self._tlv = EF_DIR.ApplicationTemplate
+
+# TS 102 221 Section 13.2
+class EF_ICCID(TransparentEF):
+    def __init__(self, fid='2fe2', sfid=0x02, name='EF.ICCID', desc='ICC Identification'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(10, 10))
+
+    def _decode_hex(self, raw_hex):
+        return {'iccid': dec_iccid(raw_hex)}
+
+    def _encode_hex(self, abstract):
+        return enc_iccid(abstract['iccid'])
+
+# TS 102 221 Section 13.3
+class EF_PL(TransRecEF):
+    def __init__(self, fid='2f05', sfid=0x05, name='EF.PL', desc='Preferred Languages'):
+        super().__init__(fid, sfid=sfid, name=name,
+                         desc=desc, rec_len=2, size=(2, None))
+
+    def _decode_record_bin(self, bin_data):
+        if bin_data == b'\xff\xff':
+            return None
+        else:
+            return bin_data.decode('ascii')
+
+    def _encode_record_bin(self, in_json):
+        if in_json is None:
+            return b'\xff\xff'
+        else:
+            return in_json.encode('ascii')
+
+
+# TS 102 221 Section 13.4
+class EF_ARR(LinFixedEF):
+    def __init__(self, fid='2f06', sfid=0x06, name='EF.ARR', desc='Access Rule Reference'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc)
+        # add those commands to the general commands of a TransparentEF
+        self.shell_commands += [self.AddlShellCommands()]
+
+    @staticmethod
+    def flatten(inp: list):
+        """Flatten the somewhat deep/complex/nested data returned from decoder."""
+        def sc_abbreviate(sc):
+            if 'always' in sc:
+                return 'always'
+            elif 'never' in sc:
+                return 'never'
+            elif 'control_reference_template' in sc:
+                return sc['control_reference_template']
+            else:
+                return sc
+
+        by_mode = {}
+        for t in inp:
+            am = t[0]
+            sc = t[1]
+            sc_abbr = sc_abbreviate(sc)
+            if 'access_mode' in am:
+                for m in am['access_mode']:
+                    by_mode[m] = sc_abbr
+            elif 'command_header' in am:
+                ins = am['command_header']['INS']
+                if 'CLA' in am['command_header']:
+                    cla = am['command_header']['CLA']
+                else:
+                    cla = None
+                cmd = ts_102_22x_cmdset.lookup(ins, cla)
+                if cmd:
+                    name = cmd.name.lower().replace(' ', '_')
+                    by_mode[name] = sc_abbr
+                else:
+                    raise ValueError
+            else:
+                raise ValueError
+        return by_mode
+
+    def _decode_record_bin(self, raw_bin_data):
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        dec = arr_seq.decode_multi(raw_bin_data)
+        # we cannot pass the result through flatten() here, as we don't have a related
+        # 'un-flattening' decoder, and hence would be unable to encode :(
+        return dec[0]
+
+    def _encode_record_bin(self, in_json):
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        return arr_seq.encode_multi(in_json)
+
+    @with_default_category('File-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        def __init__(self):
+            super().__init__()
+
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
+        def do_read_arr_record(self, opts):
+            """Read one EF.ARR record in flattened, human-friendly form."""
+            (data, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
+            data = self._cmd.lchan.selected_file.flatten(data)
+            self._cmd.poutput_json(data, opts.oneline)
+
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
+        def do_read_arr_records(self, opts):
+            """Read + decode all EF.ARR records in flattened, human-friendly form."""
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
+            # collect all results in list so they are rendered as JSON list when printing
+            data_list = []
+            for recnr in range(1, 1 + num_of_rec):
+                (data, sw) = self._cmd.lchan.read_record_dec(recnr)
+                data = self._cmd.lchan.selected_file.flatten(data)
+                data_list.append(data)
+            self._cmd.poutput_json(data_list, opts.oneline)
+
+
+# TS 102 221 Section 13.6
+class EF_UMPC(TransparentEF):
+    def __init__(self, fid='2f08', sfid=0x08, name='EF.UMPC', desc='UICC Maximum Power Consumption'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(5, 5))
+        addl_info = FlagsEnum(Byte, req_inc_idle_current=1,
+                              support_uicc_suspend=2)
+        self._construct = Struct(
+            'max_current_mA'/Int8ub, 't_op_s'/Int8ub, 'addl_info'/addl_info)
+
+
+class CardProfileUICC(CardProfile):
+
+    ORDER = 1
+
+    def __init__(self, name='UICC'):
+        files = [
+            EF_DIR(),
+            EF_ICCID(),
+            EF_PL(),
+            EF_ARR(),
+            # FIXME: DF.CD
+            EF_UMPC(),
+        ]
+        sw = {
+            'Normal': {
+                '9000': 'Normal ending of the command',
+                '91xx': 'Normal ending of the command, with extra information from the proactive UICC containing a command for the terminal',
+                '92xx': 'Normal ending of the command, with extra information concerning an ongoing data transfer session',
+            },
+            'Postponed processing': {
+                '9300': 'SIM Application Toolkit is busy. Command cannot be executed at present, further normal commands are allowed',
+            },
+            'Warnings': {
+                '6200': 'No information given, state of non-volatile memory unchanged',
+                '6281': 'Part of returned data may be corrupted',
+                '6282': 'End of file/record reached before reading Le bytes or unsuccessful search',
+                '6283': 'Selected file invalidated',
+                '6284': 'Selected file in termination state',
+                '62f1': 'More data available',
+                '62f2': 'More data available and proactive command pending',
+                '62f3': 'Response data available',
+                '63f1': 'More data expected',
+                '63f2': 'More data expected and proactive command pending',
+                '63cx': 'Command successful but after using an internal update retry routine X times',
+            },
+            'Execution errors': {
+                '6400': 'No information given, state of non-volatile memory unchanged',
+                '6500': 'No information given, state of non-volatile memory changed',
+                '6581': 'Memory problem',
+            },
+            'Checking errors': {
+                '6700': 'Wrong length',
+                '67xx': 'The interpretation of this status word is command dependent',
+                '6b00': 'Wrong parameter(s) P1-P2',
+                '6d00': 'Instruction code not supported or invalid',
+                '6e00': 'Class not supported',
+                '6f00': 'Technical problem, no precise diagnosis',
+                '6fxx': 'The interpretation of this status word is command dependent',
+            },
+            'Functions in CLA not supported': {
+                '6800': 'No information given',
+                '6881': 'Logical channel not supported',
+                '6882': 'Secure messaging not supported',
+            },
+            'Command not allowed': {
+                '6900': 'No information given',
+                '6981': 'Command incompatible with file structure',
+                '6982': 'Security status not satisfied',
+                '6983': 'Authentication/PIN method blocked',
+                '6984': 'Referenced data invalidated',
+                '6985': 'Conditions of use not satisfied',
+                '6986': 'Command not allowed (no EF selected)',
+                '6989': 'Command not allowed - secure channel - security not satisfied',
+            },
+            'Wrong parameters': {
+                '6a80': 'Incorrect parameters in the data field',
+                '6a81': 'Function not supported',
+                '6a82': 'File not found',
+                '6a83': 'Record not found',
+                '6a84': 'Not enough memory space',
+                '6a86': 'Incorrect parameters P1 to P2',
+                '6a87': 'Lc inconsistent with P1 to P2',
+                '6a88': 'Referenced data not found',
+            },
+            'Application errors': {
+                '9850': 'INCREASE cannot be performed, max value reached',
+                '9862': 'Authentication error, application specific',
+                '9863': 'Security session or association expired',
+                '9864': 'Minimum UICC suspension time is too long',
+            },
+        }
+
+        super().__init__(name, desc='ETSI TS 102 221', cla="00",
+                         sel_ctrl="0004", files_in_mf=files, sw=sw)
+
+    @staticmethod
+    def decode_select_response(resp_hex: str) -> object:
+        """ETSI TS 102 221 Section 11.1.1.3"""
+        t = FcpTemplate()
+        t.from_tlv(h2b(resp_hex))
+        d = t.to_dict()
+        return flatten_dict_lists(d['fcp_template'])
+
+    @staticmethod
+    def match_with_card(scc: SimCardCommands) -> bool:
+        return match_uicc(scc)
+
+
+class CardProfileUICCSIM(CardProfileUICC):
+    """Same as above, but including 2G SIM support"""
+
+    ORDER = 0
+
+    def __init__(self):
+        super().__init__('UICC-SIM')
+
+        # Add GSM specific files
+        self.files_in_mf.append(DF_TELECOM())
+        self.files_in_mf.append(DF_GSM())
+
+    @staticmethod
+    def match_with_card(scc: SimCardCommands) -> bool:
+        return match_uicc(scc) and match_sim(scc)

pySim/ts_102_222.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+
+# Interactive shell for working with SIM / UICC / USIM / ISIM cards
+#
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import List
+
+import cmd2
+from cmd2 import CommandSet, with_default_category, with_argparser
+import argparse
+
+from pySim.ts_31_102 import EF_UST_map, EF_USIM_ADF_map
+from pySim.ts_31_103 import EF_IST_map, EF_ISIM_ADF_map
+
+from pySim.exceptions import *
+from pySim.utils import h2b, swap_nibbles, b2h, JsonEncoder
+
+from pySim.ts_102_221 import *
+
+@with_default_category('TS 102 222 Administrative Commands')
+class Ts102222Commands(CommandSet):
+    """Administrative commands for telecommunication applications."""
+
+    def __init__(self):
+        super().__init__()
+
+    delfile_parser = argparse.ArgumentParser()
+    delfile_parser.add_argument('--force-delete', action='store_true',
+            help='I really want to permanently delete the file. I know pySim cannot re-create it yet!')
+    delfile_parser.add_argument('NAME', type=str, help='File name or FID to delete')
+
+    @cmd2.with_argparser(delfile_parser)
+    def do_delete_file(self, opts):
+        """Delete the specified file. DANGEROUS!  See TS 102 222 Section 6.4.
+        This will permanently delete the specified file from the card.
+        pySim has no support to re-create files yet, and even if it did, your card may not allow it!"""
+        if not opts.force_delete:
+            self._cmd.perror("Refusing to permanently delete the file, please read the help text.")
+            return
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        (data, sw) = self._cmd.card._scc.delete_file(f.fid)
+
+    def complete_delete_file(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for DELETE FILE"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    termdf_parser = argparse.ArgumentParser()
+    termdf_parser.add_argument('--force', action='store_true',
+            help='I really want to terminate the file. I know I can not recover from it!')
+    termdf_parser.add_argument('NAME', type=str, help='File name or FID')
+
+    @cmd2.with_argparser(termdf_parser)
+    def do_terminate_df(self, opts):
+        """Terminate the specified DF. DANGEROUS!  See TS 102 222 6.7.
+        This is a permanent, one-way operation on the card. There is no undo, you can not recover
+        a terminated DF.  The only permitted command for a terminated DF is the DLETE FILE command."""
+        if not opts.force:
+            self._cmd.perror("Refusing to terminate the file, please read the help text.")
+            return
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        (data, sw) = self._cmd.card._scc.terminate_df(f.fid)
+
+    def complete_terminate_df(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for TERMINATE DF"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    @cmd2.with_argparser(termdf_parser)
+    def do_terminate_ef(self, opts):
+        """Terminate the specified EF. DANGEROUS!  See TS 102 222 6.8.
+        This is a permanent, one-way operation on the card. There is no undo, you can not recover
+        a terminated EF.  The only permitted command for a terminated EF is the DLETE FILE command."""
+        if not opts.force:
+            self._cmd.perror("Refusing to terminate the file, please read the help text.")
+            return
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        (data, sw) = self._cmd.card._scc.terminate_ef(f.fid)
+
+    def complete_terminate_ef(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for TERMINATE EF"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    tcard_parser = argparse.ArgumentParser()
+    tcard_parser.add_argument('--force-terminate-card', action='store_true',
+            help='I really want to permanently terminate the card. It will not be usable afterwards!')
+
+    @cmd2.with_argparser(tcard_parser)
+    def do_terminate_card_usage(self, opts):
+        """Terminate the Card. SUPER DANGEROUS!  See TS 102 222 Section 6.9.
+        This will permanently brick the card and can NOT be recovered from!"""
+        if not opts.force_terminate_card:
+            self._cmd.perror("Refusing to permanently terminate the card, please read the help text.")
+            return
+        (data, sw) = self._cmd.card._scc.terminate_card_usage()
+
+    create_parser = argparse.ArgumentParser()
+    create_parser.add_argument('FILE_ID', type=str, help='File Identifier as 4-character hex string')
+    create_parser._action_groups.pop()
+    create_required = create_parser.add_argument_group('required arguments')
+    create_optional = create_parser.add_argument_group('optional arguments')
+    create_required.add_argument('--ef-arr-file-id', required=True, type=str, help='Referenced Security: File Identifier of EF.ARR')
+    create_required.add_argument('--ef-arr-record-nr', required=True, type=int, help='Referenced Security: Record Number within EF.ARR')
+    create_required.add_argument('--file-size', required=True, type=int, help='Size of file in octets')
+    create_required.add_argument('--structure', required=True, type=str, choices=['transparent', 'linear_fixed', 'ber_tlv'],
+                                 help='Structure of the to-be-created EF')
+    create_optional.add_argument('--short-file-id', type=str, help='Short File Identifier as 2-digit hex string')
+    create_optional.add_argument('--shareable', action='store_true', help='Should the file be shareable?')
+    create_optional.add_argument('--record-length', type=int, help='Length of each record in octets')
+
+    @cmd2.with_argparser(create_parser)
+    def do_create_ef(self, opts):
+        """Create a new EF below the currently selected DF.  Requires related privileges."""
+        file_descriptor = {
+            'file_descriptor_byte': {
+                'shareable': opts.shareable,
+                'file_type': 'working_ef',
+                'structure': opts.structure,
+            }
+        }
+        if opts.structure == 'linear_fixed':
+            if not opts.record_length:
+                self._cmd.perror("you must specify the --record-length for linear fixed EF")
+                return
+            file_descriptor['record_len'] = opts.record_length
+            file_descriptor['num_of_rec'] = opts.file_size // opts.record_length
+            if file_descriptor['num_of_rec'] * file_descriptor['record_len'] != opts.file_size:
+                raise ValueError("File size not evenly divisible by record length")
+        elif opts.structure == 'ber_tlv':
+            self._cmd.perror("BER-TLV creation not yet fully supported, sorry")
+            return
+        ies = [FileDescriptor(decoded=file_descriptor), FileIdentifier(decoded=opts.FILE_ID),
+               LifeCycleStatusInteger(decoded='operational_activated'),
+               SecurityAttribReferenced(decoded={'ef_arr_file_id': opts.ef_arr_file_id,
+                                                 'ef_arr_record_nr': opts.ef_arr_record_nr }),
+               FileSize(decoded=opts.file_size),
+               ShortFileIdentifier(decoded=opts.short_file_id),
+            ]
+        fcp = FcpTemplate(children=ies)
+        (data, sw) = self._cmd.card._scc.create_file(b2h(fcp.to_tlv()))
+        # the newly-created file is automatically selected but our runtime state knows nothing of it
+        self._cmd.lchan.select_file(self._cmd.lchan.selected_file)
+
+    createdf_parser = argparse.ArgumentParser()
+    createdf_parser.add_argument('FILE_ID', type=str, help='File Identifier as 4-character hex string')
+    createdf_parser._action_groups.pop()
+    createdf_required = createdf_parser.add_argument_group('required arguments')
+    createdf_optional = createdf_parser.add_argument_group('optional arguments')
+    createdf_sja_optional = createdf_parser.add_argument_group('sysmoISIM-SJA optional arguments')
+    createdf_required.add_argument('--ef-arr-file-id', required=True, type=str, help='Referenced Security: File Identifier of EF.ARR')
+    createdf_required.add_argument('--ef-arr-record-nr', required=True, type=int, help='Referenced Security: Record Number within EF.ARR')
+    createdf_optional.add_argument('--shareable', action='store_true', help='Should the file be shareable?')
+    createdf_optional.add_argument('--aid', type=str, help='Application ID (creates an ADF, instead of a DF)')
+    # mandatory by spec, but ignored by several OS, so don't force the user
+    createdf_optional.add_argument('--total-file-size', type=int, help='Physical memory allocated for DF/ADi in octets')
+    createdf_sja_optional.add_argument('--permit-rfm-create', action='store_true')
+    createdf_sja_optional.add_argument('--permit-rfm-delete-terminate', action='store_true')
+    createdf_sja_optional.add_argument('--permit-other-applet-create', action='store_true')
+    createdf_sja_optional.add_argument('--permit-other-applet-delete-terminate', action='store_true')
+
+    @cmd2.with_argparser(createdf_parser)
+    def do_create_df(self, opts):
+        """Create a new DF below the currently selected DF.  Requires related privileges."""
+        file_descriptor = {
+            'file_descriptor_byte': {
+                'shareable': opts.shareable,
+                'file_type': 'df',
+                'structure': 'no_info_given',
+            }
+        }
+        ies = []
+        ies.append(FileDescriptor(decoded=file_descriptor))
+        ies.append(FileIdentifier(decoded=opts.FILE_ID))
+        if opts.aid:
+            ies.append(DfName(decoded=opts.aid))
+        ies.append(LifeCycleStatusInteger(decoded='operational_activated'))
+        ies.append(SecurityAttribReferenced(decoded={'ef_arr_file_id': opts.ef_arr_file_id,
+                                                     'ef_arr_record_nr': opts.ef_arr_record_nr }))
+        if opts.total_file_size:
+            ies.append(TotalFileSize(decoded=opts.total_file_size))
+        # TODO: Spec states PIN Status Template DO is mandatory
+        if opts.permit_rfm_create or opts.permit_rfm_delete_terminate or opts.permit_other_applet_create or opts.permit_other_applet_delete_terminate:
+               toolkit_ac = {
+                   'rfm_create': opts.permit_rfm_create,
+                   'rfm_delete_terminate': opts.permit_rfm_delete_terminate,
+                   'other_applet_create': opts.permit_other_applet_create,
+                   'other_applet_delete_terminate': opts.permit_other_applet_delete_terminate,
+                   }
+               ies.append(ProprietaryInformation(children=[ToolkitAccessConditions(decoded=toolkit_ac)]))
+        fcp = FcpTemplate(children=ies)
+        (data, sw) = self._cmd.card._scc.create_file(b2h(fcp.to_tlv()))
+        # the newly-created file is automatically selected but our runtime state knows nothing of it
+        self._cmd.lchan.select_file(self._cmd.lchan.selected_file)

pySim/ts_31_102_telecom.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+DF_PHONEBOOK, DF_MULTIMEDIA, DF_MCS as specified in 3GPP TS 31.102 V16.6.0
+Needs to be a separate python module to avoid cyclic imports
+"""
+
+#
+# Copyright (C) 2022 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from pySim.tlv import *
+from pySim.filesystem import *
+from pySim.construct import *
+from construct import Optional as COptional
+from construct import *
+
+# TS 31.102 Section 4.2.8
+class EF_UServiceTable(TransparentEF):
+    def __init__(self, fid, sfid, name, desc, size, table, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
+        self.table = table
+
+    @staticmethod
+    def _bit_byte_offset_for_service(service: int) -> Tuple[int, int]:
+        i = service - 1
+        byte_offset = i//8
+        bit_offset = (i % 8)
+        return (byte_offset, bit_offset)
+
+    def _decode_bin(self, in_bin):
+        ret = {}
+        for i in range(0, len(in_bin)):
+            byte = in_bin[i]
+            for bitno in range(0, 8):
+                service_nr = i * 8 + bitno + 1
+                ret[service_nr] = {
+                    'activated': True if byte & (1 << bitno) else False
+                }
+                if service_nr in self.table:
+                    ret[service_nr]['description'] = self.table[service_nr]
+        return ret
+
+    def _encode_bin(self, in_json):
+        # compute the required binary size
+        bin_len = 0
+        for srv in in_json.keys():
+            service_nr = int(srv)
+            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
+                service_nr)
+            if byte_offset >= bin_len:
+                bin_len = byte_offset+1
+        # encode the actual data
+        out = bytearray(b'\x00' * bin_len)
+        for srv in in_json.keys():
+            service_nr = int(srv)
+            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
+                service_nr)
+            if in_json[srv]['activated'] == True:
+                bit = 1
+            else:
+                bit = 0
+            out[byte_offset] |= (bit) << bit_offset
+        return out
+
+    def get_active_services(self, cmd):
+        # obtain list of currently active services
+        (service_data, sw) = cmd.lchan.read_binary_dec()
+        active_services = []
+        for s in service_data.keys():
+            if service_data[s]['activated']:
+                active_services.append(s)
+        return active_services
+
+    def ust_service_check(self, cmd):
+        """Check consistency between services of this file and files present/activated"""
+        num_problems = 0
+        # obtain list of currently active services
+        active_services = self.get_active_services(cmd)
+        # iterate over all the service-constraints we know of
+        files_by_service = self.parent.files_by_service
+        try:
+            for s in sorted(files_by_service.keys()):
+                active_str = 'active' if s in active_services else 'inactive'
+                cmd.poutput("Checking service No %u (%s)" % (s, active_str))
+                for f in files_by_service[s]:
+                    should_exist = f.should_exist_for_services(active_services)
+                    try:
+                        cmd.lchan.select_file(f)
+                        sw = None
+                        exists = True
+                    except SwMatchError as e:
+                        sw = str(e)
+                        exists = False
+                    if exists != should_exist:
+                        num_problems += 1
+                        if exists:
+                            cmd.perror("  ERROR: File %s is selectable but should not!" % f)
+                        else:
+                            cmd.perror("  ERROR: File %s is not selectable (%s) but should!" %  (f, sw))
+        finally:
+            # re-select the EF.UST
+            cmd.lchan.select_file(self)
+        return num_problems
+
+
+
+# TS 31.102 Section 4.4.2.1
+class EF_PBR(LinFixedEF):
+    def __init__(self, fid='4F30', name='EF.PBR', desc='Phone Book Reference', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+        #self._tlv = FIXME
+
+# TS 31.102 Section 4.4.2.12.2
+class EF_PSC(TransparentEF):
+    _construct = Struct('synce_counter'/Int32ub)
+    def __init__(self, fid='4F22', name='EF.PSC', desc='Phone Book Synchronization Counter', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+        #self._tlv = FIXME
+
+# TS 31.102 Section 4.4.2.12.3
+class EF_CC(TransparentEF):
+    _construct = Struct('change_counter'/Int16ub)
+    def __init__(self, fid='4F23', name='EF.CC', desc='Change Counter', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.4.2.12.4
+class EF_PUID(TransparentEF):
+    _construct = Struct('previous_uid'/Int16ub)
+    def __init__(self, fid='4F24', name='EF.PUID', desc='Previous Unique Identifer', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.4.2
+class DF_PHONEBOOK(CardDF):
+    def __init__(self, fid='5F3A', name='DF.PHONEBOOK', desc='Phonebook', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_PBR(),
+            EF_PSC(),
+            EF_CC(),
+            EF_PUID(),
+            # FIXME: Those 4Fxx entries with unspecified FID...
+            ]
+        self.add_files(files)
+
+
+
+# TS 31.102 Section 4.6.3.1
+class EF_MML(BerTlvEF):
+    def __init__(self, fid='4F47', name='EF.MML', desc='Multimedia Messages List', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.6.3.2
+class EF_MMDF(BerTlvEF):
+    def __init__(self, fid='4F48', name='EF.MMDF', desc='Multimedia Messages Data File', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+class DF_MULTIMEDIA(CardDF):
+    def __init__(self, fid='5F3B', name='DF.MULTIMEDIA', desc='Multimedia', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_MML(),
+            EF_MMDF(),
+            ]
+        self.add_files(files)
+
+
+# TS 31.102 Section 4.6.4.1
+EF_MST_map = {
+    1: 'MCPTT UE configuration data',
+    2: 'MCPTT User profile data',
+    3: 'MCS Group configuration data',
+    4: 'MCPTT Service configuration data',
+    5: 'MCS UE initial configuration data',
+    6: 'MCData UE configuration data',
+    7: 'MCData user profile data',
+    8: 'MCData service configuration data',
+    9: 'MCVideo UE configuration data',
+    10: 'MCVideo user profile data',
+    11: 'MCVideo service configuration data',
+    }
+
+# TS 31.102 Section 4.6.4.2
+class EF_MCS_CONFIG(BerTlvEF):
+    class McpttUeConfigurationData(BER_TLV_IE, tag=0x80):
+        pass
+    class McpttUserProfileData(BER_TLV_IE, tag=0x81):
+        pass
+    class McsGroupConfigurationData(BER_TLV_IE, tag=0x82):
+        pass
+    class McpttServiceConfigurationData(BER_TLV_IE, tag=0x83):
+        pass
+    class McsUeInitialConfigurationData(BER_TLV_IE, tag=0x84):
+        pass
+    class McdataUeConfigurationData(BER_TLV_IE, tag=0x85):
+        pass
+    class McdataUserProfileData(BER_TLV_IE, tag=0x86):
+        pass
+    class McdataServiceConfigurationData(BER_TLV_IE, tag=0x87):
+        pass
+    class McvideoUeConfigurationData(BER_TLV_IE, tag=0x88):
+        pass
+    class McvideoUserProfileData(BER_TLV_IE, tag=0x89):
+        pass
+    class McvideoServiceConfigurationData(BER_TLV_IE, tag=0x8a):
+        pass
+    class McsConfigDataCollection(TLV_IE_Collection, nested=[McpttUeConfigurationData,
+                                  McpttUserProfileData, McsGroupConfigurationData,
+                                  McpttServiceConfigurationData, McsUeInitialConfigurationData,
+                                  McdataUeConfigurationData, McdataUserProfileData,
+                                  McdataServiceConfigurationData, McvideoUeConfigurationData,
+                                  McvideoUserProfileData, McvideoServiceConfigurationData]):
+        pass
+    def __init__(self, fid='4F02', sfid=0x02, name='EF.MCS_CONFIG', desc='MCS configuration data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_MCS_CONFIG.McsConfigDataCollection
+
+# TS 31.102 Section 4.6.4.1
+class EF_MST(EF_UServiceTable):
+    def __init__(self, fid='4F01', sfid=0x01, name='EF.MST', desc='MCS Service Table', size=(2,2),
+                 table=EF_MST_map, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, table=table)
+
+class DF_MCS(CardDF):
+    def __init__(self, fid='5F3D', name='DF.MCS', desc='Mission Critical Services', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_MST(),
+            EF_MCS_CONFIG(),
+            ]
+        self.add_files(files)
+
+
+# TS 31.102 Section 4.6.5.2
+EF_VST_map = {
+    1: 'MCPTT UE configuration data',
+    2: 'MCPTT User profile data',
+    3: 'MCS Group configuration data',
+    4: 'MCPTT Service configuration data',
+    5: 'MCS UE initial configuration data',
+    6: 'MCData UE configuration data',
+    7: 'MCData user profile data',
+    8: 'MCData service configuration data',
+    9: 'MCVideo UE configuration data',
+    10: 'MCVideo user profile data',
+    11: 'MCVideo service configuration data',
+    }
+
+# TS 31.102 Section 4.6.5.2
+class EF_VST(EF_UServiceTable):
+    def __init__(self, fid='4F01', sfid=0x01, name='EF.VST', desc='V2X Service Table', size=(2,2),
+                 table=EF_VST_map, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, table=table)
+
+# TS 31.102 Section 4.6.5.3
+class EF_V2X_CONFIG(BerTlvEF):
+    class V2xConfigurationData(BER_TLV_IE, tag=0x80):
+        pass
+    class V2xConfigDataCollection(TLV_IE_Collection, nested=[V2xConfigurationData]):
+        pass
+    def __init__(self, fid='4F02', sfid=0x02, name='EF.V2X_CONFIG', desc='V2X configuration data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_V2X_CONFIG.V2xConfigDataCollection
+
+# TS 31.102 Section 4.6.5
+class DF_V2X(CardDF):
+    def __init__(self, fid='5F3E', name='DF.V2X', desc='Vehicle to X', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_VST(),
+            EF_V2X_CONFIG(),
+            ]
+        self.add_files(files)

pySim/ts_31_103.py

@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+
+"""
+Various constants from 3GPP TS 31.103 V16.1.0
+"""
+
+#
+# Copyright (C) 2020 Supreeth Herle <herlesupreeth@gmail.com>
+# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from pySim.filesystem import *
+from pySim.utils import *
+from pySim.tlv import *
+from pySim.ts_51_011 import EF_AD, EF_SMS, EF_SMSS, EF_SMSR, EF_SMSP
+from pySim.ts_31_102 import ADF_USIM, EF_FromPreferred
+from pySim.ts_31_102_telecom import EF_UServiceTable
+import pySim.ts_102_221
+from pySim.ts_102_221 import EF_ARR
+
+# Mapping between ISIM Service Number and its description
+EF_IST_map = {
+    1: 'P-CSCF address',
+    2: 'Generic Bootstrapping Architecture (GBA)',
+    3: 'HTTP Digest',
+    4: 'GBA-based Local Key Establishment Mechanism',
+    5: 'Support of P-CSCF discovery for IMS Local Break Out',
+    6: 'Short Message Storage (SMS)',
+    7: 'Short Message Status Reports (SMSR)',
+    8: 'Support for SM-over-IP including data download via SMS-PP as defined in TS 31.111 [31]',
+    9: 'Communication Control for IMS by ISIM',
+    10: 'Support of UICC access to IMS',
+    11: 'URI support by UICC',
+    12: 'Media Type support',
+    13: 'IMS call disconnection cause',
+    14: 'URI support for MO SHORT MESSAGE CONTROL',
+    15: 'MCPTT',
+    16: 'URI support for SMS-PP DOWNLOAD as defined in 3GPP TS 31.111 [31]',
+    17: 'From Preferred',
+    18: 'IMS configuration data',
+    19: 'XCAP Configuration Data',
+    20: 'WebRTC URI',
+    21: 'MuD and MiD configuration data',
+}
+
+EF_ISIM_ADF_map = {
+    'IST': '6F07',
+    'IMPI': '6F02',
+    'DOMAIN': '6F03',
+    'IMPU': '6F04',
+    'AD': '6FAD',
+    'ARR': '6F06',
+    'PCSCF': '6F09',
+    'GBAP': '6FD5',
+    'GBANL': '6FD7',
+    'NAFKCA': '6FDD',
+    'UICCIARI': '6FE7',
+    'SMS': '6F3C',
+    'SMSS': '6F43',
+    'SMSR': '6F47',
+    'SMSP': '6F42',
+    'FromPreferred': '6FF7',
+    'IMSConfigData': '6FF8',
+    'XCAPConfigData': '6FFC',
+    'WebRTCURI': '6FFA'
+}
+
+# TS 31.103 Section 4.2.2
+class EF_IMPI(TransparentEF):
+    class nai(BER_TLV_IE, tag=0x80):
+        _construct = GreedyString("utf8")
+
+    def __init__(self, fid='6f02', sfid=0x02, name='EF.IMPI', desc='IMS private user identity', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_IMPI.nai
+
+# TS 31.103 Section 4.2.3
+class EF_DOMAIN(TransparentEF):
+    class domain(BER_TLV_IE, tag=0x80):
+        _construct = GreedyString("utf8")
+
+    def __init__(self, fid='6f03', sfid=0x05, name='EF.DOMAIN', desc='Home Network Domain Name', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_DOMAIN.domain
+
+# TS 31.103 Section 4.2.4
+class EF_IMPU(LinFixedEF):
+    class impu(BER_TLV_IE, tag=0x80):
+        _construct = GreedyString("utf8")
+
+    def __init__(self, fid='6f04', sfid=0x04, name='EF.IMPU', desc='IMS public user identity', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_IMPU.impu
+
+# TS 31.103 Section 4.2.7
+class EF_IST(EF_UServiceTable):
+    def __init__(self, **kwargs):
+        super().__init__('6f07', 0x07, 'EF.IST', 'ISIM Service Table', (1, None), EF_IST_map)
+        # add those commands to the general commands of a TransparentEF
+        self.shell_commands += [self.AddlShellCommands()]
+
+    @with_default_category('File-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        def __init__(self):
+            super().__init__()
+
+        def do_ist_service_activate(self, arg):
+            """Activate a service within EF.IST"""
+            self._cmd.card.update_ist(int(arg), 1)
+
+        def do_ist_service_deactivate(self, arg):
+            """Deactivate a service within EF.IST"""
+            self._cmd.card.update_ist(int(arg), 0)
+
+        def do_ist_service_check(self, arg):
+            """Check consistency between services of this file and files present/activated.
+
+            Many services determine if one or multiple files shall be present/activated or if they shall be
+            absent/deactivated.  This performs a consistency check to ensure that no services are activated
+            for files that are not - and vice-versa, no files are activated for services that are not.  Error
+            messages are printed for every inconsistency found."""
+            selected_file = self._cmd.lchan.selected_file
+            num_problems = selected_file.ust_service_check(self._cmd)
+            self._cmd.poutput("===> %u service / file inconsistencies detected" % num_problems)
+
+
+# TS 31.103 Section 4.2.8
+class EF_PCSCF(LinFixedEF):
+    def __init__(self, fid='6f09', sfid=None, name='EF.P-CSCF', desc='P-CSCF Address', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+
+    def _decode_record_hex(self, raw_hex):
+        addr, addr_type = dec_addr_tlv(raw_hex)
+        return {"addr": addr, "addr_type": addr_type}
+
+    def _encode_record_hex(self, json_in):
+        addr = json_in['addr']
+        addr_type = json_in['addr_type']
+        return enc_addr_tlv(addr, addr_type)
+
+# TS 31.103 Section 4.2.9
+class EF_GBABP(TransparentEF):
+    def __init__(self, fid='6fd5', sfid=None, name='EF.GBABP', desc='GBA Bootstrapping', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+
+# TS 31.103 Section 4.2.10
+class EF_GBANL(LinFixedEF):
+    def __init__(self, fid='6fd7', sfid=None, name='EF.GBANL', desc='GBA NAF List', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+
+# TS 31.103 Section 4.2.11
+class EF_NAFKCA(LinFixedEF):
+    def __init__(self, fid='6fdd', sfid=None, name='EF.NAFKCA', desc='NAF Key Centre Address', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+
+# TS 31.103 Section 4.2.16
+class EF_UICCIARI(LinFixedEF):
+    class iari(BER_TLV_IE, tag=0x80):
+        _construct = GreedyString("utf8")
+
+    def __init__(self, fid='6fe7', sfid=None, name='EF.UICCIARI', desc='UICC IARI', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_UICCIARI.iari
+
+# TS 31.103 Section 4.2.18
+class EF_IMSConfigData(BerTlvEF):
+    class ImsConfigDataEncoding(BER_TLV_IE, tag=0x80):
+        _construct = HexAdapter(Bytes(1))
+    class ImsConfigData(BER_TLV_IE, tag=0x81):
+        _construct = GreedyString
+    # pylint: disable=undefined-variable
+    class ImsConfigDataCollection(TLV_IE_Collection, nested=[ImsConfigDataEncoding, ImsConfigData]):
+        pass
+    def __init__(self, fid='6ff8', sfid=None, name='EF.IMSConfigData', desc='IMS Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_IMSConfigData.ImsConfigDataCollection
+
+# TS 31.103 Section 4.2.19
+class EF_XCAPConfigData(BerTlvEF):
+    class Access(BER_TLV_IE, tag=0x81):
+        pass
+    class ApplicationName(BER_TLV_IE, tag=0x82):
+        pass
+    class ProviderID(BER_TLV_IE, tag=0x83):
+        pass
+    class URI(BER_TLV_IE, tag=0x84):
+        pass
+    class XcapAuthenticationUserName(BER_TLV_IE, tag=0x85):
+        pass
+    class XcapAuthenticationPassword(BER_TLV_IE, tag=0x86):
+        pass
+    class XcapAuthenticationType(BER_TLV_IE, tag=0x87):
+        pass
+    class AddressType(BER_TLV_IE, tag=0x88):
+        pass
+    class Address(BER_TLV_IE, tag=0x89):
+        pass
+    class PDPAuthenticationType(BER_TLV_IE, tag=0x8a):
+        pass
+    class PDPAuthenticationName(BER_TLV_IE, tag=0x8b):
+        pass
+    class PDPAuthenticationSecret(BER_TLV_IE, tag=0x8c):
+        pass
+
+    class AccessForXCAP(BER_TLV_IE, tag=0x81):
+        pass
+    class NumberOfXcapConnParPolicy(BER_TLV_IE, tag=0x82):
+        _construct = Int8ub
+    # pylint: disable=undefined-variable
+    class XcapConnParamsPolicyPart(BER_TLV_IE, tag=0xa1, nested=[Access, ApplicationName, ProviderID, URI,
+                                 XcapAuthenticationUserName, XcapAuthenticationPassword,
+                                 XcapAuthenticationType, AddressType, Address, PDPAuthenticationType,
+                                 PDPAuthenticationName, PDPAuthenticationSecret]):
+        pass
+    class XcapConnParamsPolicy(BER_TLV_IE, tag=0xa0, nested=[AccessForXCAP, NumberOfXcapConnParPolicy, XcapConnParamsPolicyPart]):
+        pass
+    class XcapConnParamsPolicyDO(BER_TLV_IE, tag=0x80, nested=[XcapConnParamsPolicy]):
+        pass
+    def __init__(self, fid='6ffc', sfid=None, name='EF.XCAPConfigData', desc='XCAP Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_XCAPConfigData.XcapConnParamsPolicy
+
+# TS 31.103 Section 4.2.20
+class EF_WebRTCURI(TransparentEF):
+    class uri(BER_TLV_IE, tag=0x80):
+        _construct = GreedyString("utf8")
+
+    def __init__(self, fid='6ffa', sfid=None, name='EF.WebRTCURI', desc='WebRTC URI', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_WebRTCURI.uri
+
+# TS 31.103 Section 4.2.21
+class EF_MuDMiDConfigData(BerTlvEF):
+    class MudMidConfigDataEncoding(BER_TLV_IE, tag=0x80):
+        _construct = HexAdapter(Bytes(1))
+    class MudMidConfigData(BER_TLV_IE, tag=0x81):
+        _construct = GreedyString
+    # pylint: disable=undefined-variable
+    class MudMidConfigDataCollection(TLV_IE_Collection, nested=[MudMidConfigDataEncoding, MudMidConfigData]):
+        pass
+    def __init__(self, fid='6ffe', sfid=None, name='EF.MuDMiDConfigData',
+                 desc='MuD and MiD Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_MuDMiDConfigData.MudMidConfigDataCollection
+
+
+class ADF_ISIM(CardADF):
+    def __init__(self, aid='a0000000871004', name='ADF.ISIM', fid=None, sfid=None,
+                 desc='ISIM Application'):
+        super().__init__(aid=aid, fid=fid, sfid=sfid, name=name, desc=desc)
+
+        files = [
+            EF_IMPI(),
+            EF_DOMAIN(),
+            EF_IMPU(),
+            EF_AD(),
+            EF_ARR('6f06', 0x06),
+            EF_IST(),
+            EF_PCSCF(service=5),
+            EF_GBABP(service=2),
+            EF_GBANL(service=2),
+            EF_NAFKCA(service=2),
+            EF_SMS(service=(6,8)),
+            EF_SMSS(service=(6,8)),
+            EF_SMSR(service=(7,8)),
+            EF_SMSP(service=8),
+            EF_UICCIARI(service=10),
+            EF_FromPreferred(service=17),
+            EF_IMSConfigData(service=18),
+            EF_XCAPConfigData(service=19),
+            EF_WebRTCURI(service=20),
+            EF_MuDMiDConfigData(service=21),
+        ]
+        self.add_files(files)
+        # add those commands to the general commands of a TransparentEF
+        self.shell_commands += [ADF_USIM.AddlShellCommands()]
+
+    def decode_select_response(self, data_hex):
+        return pySim.ts_102_221.CardProfileUICC.decode_select_response(data_hex)
+
+
+# TS 31.103 Section 7.1
+sw_isim = {
+    'Security management': {
+        '9862': 'Authentication error, incorrect MAC',
+        '9864': 'Authentication error, security context not supported',
+    }
+}
+
+
+class CardApplicationISIM(CardApplication):
+    def __init__(self):
+        super().__init__('ISIM', adf=ADF_ISIM(), sw=sw_isim)

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

pysim-testdata/Fairwaves-SIM.data

@@ -0,0 +1,5 @@
+MCC=001
+MNC=01
+IMSI=001010000000111
+ADM_HEX=CAE743DB9C5B5A58
+

pysim-testdata/Fairwaves-SIM.ok

@@ -0,0 +1,120 @@
+Using PC/SC reader interface
+Reading ...
+Autodetected card type: Fairwaves-SIM
+ICCID: 8988219000000117833
+IMSI: 001010000000111
+GID1: ffffffffffffffff
+GID2: ffffffffffffffff
+SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
+SPN: Fairwaves
+Show in HPLMN: False
+Hide in OPLMN: False
+PLMNsel: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+PLMNwAcT:
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+OPLMNwAcT:
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+HPLMNAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+ACC: 0008
+MSISDN: Not available
+Administrative data: 00000002
+	MS operation mode: normal
+	Ciphering Indicator: disabled
+SIM Service Table: ff3cc3ff030fff0f000fff03f0c0
+	Service 1 - CHV1 disable function
+	Service 2 - Abbreviated Dialling Numbers (ADN)
+	Service 3 - Fixed Dialling Numbers (FDN)
+	Service 4 - Short Message Storage (SMS)
+	Service 5 - Advice of Charge (AoC)
+	Service 6 - Capability Configuration Parameters (CCP)
+	Service 7 - PLMN selector
+	Service 8 - RFU
+	Service 11 - Extension2
+	Service 12 - SMS Parameters
+	Service 13 - Last Number Dialled (LND)
+	Service 14 - Cell Broadcast Message Identifier
+	Service 17 - Service Provider Name
+	Service 18 - Service Dialling Numbers (SDN)
+	Service 23 - enhanced Multi-Level Precedence and Pre-emption Service
+	Service 24 - Automatic Answer for eMLPP
+	Service 25 - Data download via SMS-CB
+	Service 26 - Data download via SMS-PP
+	Service 27 - Menu selection
+	Service 28 - Call control
+	Service 29 - Proactive SIM
+	Service 30 - Cell Broadcast Message Identifier Ranges
+	Service 31 - Barred Dialling Numbers (BDN)
+	Service 32 - Extension4
+	Service 33 - De-personalization Control Keys
+	Service 34 - Co-operative Network List
+	Service 41 - USSD string data object supported in Call Control
+	Service 42 - RUN AT COMMAND command
+	Service 43 - User controlled PLMN Selector with Access Technology
+	Service 44 - Operator controlled PLMN Selector with Access Technology
+	Service 49 - MExE
+	Service 50 - Reserved and shall be ignored
+	Service 51 - PLMN Network Name
+	Service 52 - Operator PLMN List
+	Service 53 - Mailbox Dialling Numbers
+	Service 54 - Message Waiting Indication Status
+	Service 55 - Call Forwarding Indication Status
+	Service 56 - Service Provider Display Information
+	Service 57 - Multimedia Messaging Service (MMS)
+	Service 58 - Extension 8
+	Service 59 - MMS User Connectivity Parameters
+
+USIM Service Table: 01ea1ffc21360480010000
+	Service 1 - Local Phone Book
+	Service 10 - Short Message Storage (SMS)
+	Service 12 - Short Message Service Parameters (SMSP)
+	Service 14 - Capability Configuration Parameters 2 (CCP2)
+	Service 15 - Cell Broadcast Message Identifier
+	Service 16 - Cell Broadcast Message Identifier Ranges
+	Service 17 - Group Identifier Level 1
+	Service 18 - Group Identifier Level 2
+	Service 19 - Service Provider Name
+	Service 20 - User controlled PLMN selector with Access Technology
+	Service 21 - MSISDN
+	Service 27 - GSM Access
+	Service 28 - Data download via SMS-PP
+	Service 29 - Data download via SMS-CB
+	Service 30 - Call Control by USIM
+	Service 31 - MO-SMS Control by USIM
+	Service 32 - RUN AT COMMAND command
+	Service 33 - shall be set to 1
+	Service 38 - GSM security context
+	Service 42 - Operator controlled PLMN selector with Access Technology
+	Service 43 - HPLMN selector with Access Technology
+	Service 45 - PLMN Network Name
+	Service 46 - Operator PLMN List
+	Service 51 - Service Provider Display Information
+	Service 64 - VGCS security
+	Service 65 - VBS security
+
+Done !
+

pysim-testdata/Wavemobile-SIM.data

@@ -0,0 +1,5 @@
+MCC=001
+MNC=01
+IMSI=001010000000102
+ADM_HEX=15E31383624FDC8A
+

pysim-testdata/Wavemobile-SIM.ok

@@ -0,0 +1,136 @@
+Using PC/SC reader interface
+Reading ...
+Autodetected card type: Wavemobile-SIM
+ICCID: 89445310150011013678
+IMSI: 001010000000102
+GID1: Can't read file -- SW match failed! Expected 9000 and got 6a82.
+GID2: Can't read file -- SW match failed! Expected 9000 and got 6a82.
+SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
+SPN: wavemobile
+Show in HPLMN: False
+Hide in OPLMN: False
+PLMNsel: 00f110ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+PLMNwAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+OPLMNwAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+HPLMNAcT: Can't read file -- SW match failed! Expected 9000 and got 6a82.
+ACC: abce
+MSISDN: Not available
+Administrative data: 00000102
+	MS operation mode: normal
+	Ciphering Indicator: enabled
+SIM Service Table: ff33ff0f3c00ff0f000cf0c0f0030000
+	Service 1 - CHV1 disable function
+	Service 2 - Abbreviated Dialling Numbers (ADN)
+	Service 3 - Fixed Dialling Numbers (FDN)
+	Service 4 - Short Message Storage (SMS)
+	Service 5 - Advice of Charge (AoC)
+	Service 6 - Capability Configuration Parameters (CCP)
+	Service 7 - PLMN selector
+	Service 8 - RFU
+	Service 9 - MSISDN
+	Service 10 - Extension1
+	Service 13 - Last Number Dialled (LND)
+	Service 14 - Cell Broadcast Message Identifier
+	Service 17 - Service Provider Name
+	Service 18 - Service Dialling Numbers (SDN)
+	Service 19 - Extension3
+	Service 20 - RFU
+	Service 21 - VGCS Group Identifier List (EFVGCS and EFVGCSS)
+	Service 22 - VBS Group Identifier List (EFVBS and EFVBSS)
+	Service 23 - enhanced Multi-Level Precedence and Pre-emption Service
+	Service 24 - Automatic Answer for eMLPP
+	Service 25 - Data download via SMS-CB
+	Service 26 - Data download via SMS-PP
+	Service 27 - Menu selection
+	Service 28 - Call control
+	Service 35 - Short Message Status Reports
+	Service 36 - Network's indication of alerting in the MS
+	Service 37 - Mobile Originated Short Message control by SIM
+	Service 38 - GPRS
+	Service 49 - MExE
+	Service 50 - Reserved and shall be ignored
+	Service 51 - PLMN Network Name
+	Service 52 - Operator PLMN List
+	Service 53 - Mailbox Dialling Numbers
+	Service 54 - Message Waiting Indication Status
+	Service 55 - Call Forwarding Indication Status
+	Service 56 - Service Provider Display Information
+	Service 57 - Multimedia Messaging Service (MMS)
+	Service 58 - Extension 8
+	Service 59 - MMS User Connectivity Parameters
+
+USIM Service Table: 9eff1b3c37fe5900000000
+	Service 2 - Fixed Dialling Numbers (FDN)
+	Service 3 - Extension 2
+	Service 4 - Service Dialling Numbers (SDN)
+	Service 5 - Extension3
+	Service 8 - Outgoing Call Information (OCI and OCT)
+	Service 9 - Incoming Call Information (ICI and ICT)
+	Service 10 - Short Message Storage (SMS)
+	Service 11 - Short Message Status Reports (SMSR)
+	Service 12 - Short Message Service Parameters (SMSP)
+	Service 13 - Advice of Charge (AoC)
+	Service 14 - Capability Configuration Parameters 2 (CCP2)
+	Service 15 - Cell Broadcast Message Identifier
+	Service 16 - Cell Broadcast Message Identifier Ranges
+	Service 17 - Group Identifier Level 1
+	Service 18 - Group Identifier Level 2
+	Service 20 - User controlled PLMN selector with Access Technology
+	Service 21 - MSISDN
+	Service 27 - GSM Access
+	Service 28 - Data download via SMS-PP
+	Service 29 - Data download via SMS-CB
+	Service 30 - Call Control by USIM
+	Service 33 - shall be set to 1
+	Service 34 - Enabled Services Table
+	Service 35 - APN Control List (ACL)
+	Service 37 - Co-operative Network List
+	Service 38 - GSM security context
+	Service 42 - Operator controlled PLMN selector with Access Technology
+	Service 43 - HPLMN selector with Access Technology
+	Service 44 - Extension 5
+	Service 45 - PLMN Network Name
+	Service 46 - Operator PLMN List
+	Service 47 - Mailbox Dialling Numbers
+	Service 48 - Message Waiting Indication Status
+	Service 49 - Call Forwarding Indication Status
+	Service 52 - Multimedia Messaging Service (MMS)
+	Service 53 - Extension 8
+	Service 55 - MMS User Connectivity Parameters
+
+Done !
+

pysim-testdata/fakemagicsim.data

@@ -0,0 +1,6 @@
+MCC=001
+MNC=01
+ICCID=1122334455667788990
+KI=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+OPC=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+IMSI=001010000000102

pysim-testdata/fakemagicsim.ok

@@ -0,0 +1,59 @@
+Using PC/SC reader interface
+Reading ...
+Autodetected card type: fakemagicsim
+Can't read AIDs from SIM -- SW match failed! Expected 9000 and got 9404.
+ICCID: 1122334455667788990
+IMSI: 001010000000102
+GID1: Can't read file -- SW match failed! Expected 9000 and got 9404.
+GID2: Can't read file -- SW match failed! Expected 9000 and got 9404.
+SMSP: ffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SPN: Magic
+Show in HPLMN: True
+Hide in OPLMN: False
+PLMNsel: 00f110ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+PLMNwAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
+OPLMNwAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
+HPLMNAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
+ACC: ffff
+MSISDN: Not available
+Administrative data: 000000
+	MS operation mode: normal
+	Ciphering Indicator: disabled
+SIM Service Table: ff3fff0f0300f003000c
+	Service 1 - CHV1 disable function
+	Service 2 - Abbreviated Dialling Numbers (ADN)
+	Service 3 - Fixed Dialling Numbers (FDN)
+	Service 4 - Short Message Storage (SMS)
+	Service 5 - Advice of Charge (AoC)
+	Service 6 - Capability Configuration Parameters (CCP)
+	Service 7 - PLMN selector
+	Service 8 - RFU
+	Service 9 - MSISDN
+	Service 10 - Extension1
+	Service 11 - Extension2
+	Service 12 - SMS Parameters
+	Service 13 - Last Number Dialled (LND)
+	Service 14 - Cell Broadcast Message Identifier
+	Service 17 - Service Provider Name
+	Service 18 - Service Dialling Numbers (SDN)
+	Service 19 - Extension3
+	Service 20 - RFU
+	Service 21 - VGCS Group Identifier List (EFVGCS and EFVGCSS)
+	Service 22 - VBS Group Identifier List (EFVBS and EFVBSS)
+	Service 23 - enhanced Multi-Level Precedence and Pre-emption Service
+	Service 24 - Automatic Answer for eMLPP
+	Service 25 - Data download via SMS-CB
+	Service 26 - Data download via SMS-PP
+	Service 27 - Menu selection
+	Service 28 - Call control
+	Service 33 - De-personalization Control Keys
+	Service 34 - Co-operative Network List
+	Service 53 - Mailbox Dialling Numbers
+	Service 54 - Message Waiting Indication Status
+	Service 55 - Call Forwarding Indication Status
+	Service 56 - Service Provider Display Information
+	Service 57 - Multimedia Messaging Service (MMS)
+	Service 58 - Extension 8
+
+Done !
+

pysim-testdata/sysmoISIM-SJA2.data

@@ -0,0 +1,7 @@
+MCC=001
+MNC=01
+ICCID=1122334455667788990
+KI=AABBCCDDEEFFAABBCCDDEEFFAABBCCDD
+OPC=12345678901234567890123456789012
+IMSI=001010000000102
+ADM=67225880

pysim-testdata/sysmoISIM-SJA2.ok

@@ -0,0 +1,212 @@
+Using PC/SC reader interface
+Reading ...
+Autodetected card type: sysmoISIM-SJA2
+ICCID: 8988211000000467343
+IMSI: 001010000000102
+GID1: ffffffffffffffffffff
+GID2: ffffffffffffffffffff
+SMSP: ffffffffffffffffffffffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SPN: Magic
+Show in HPLMN: True
+Hide in OPLMN: True
+PLMNsel: 00f110ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+PLMNwAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+OPLMNwAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+HPLMNAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+ACC: 0010
+MSISDN (NPI=1 ToN=3): 6766266
+Administrative data: 00000002
+	MS operation mode: normal
+	Ciphering Indicator: disabled
+SIM Service Table: ff33ffff3f003f0f300cf0c3f00000
+	Service 1 - CHV1 disable function
+	Service 2 - Abbreviated Dialling Numbers (ADN)
+	Service 3 - Fixed Dialling Numbers (FDN)
+	Service 4 - Short Message Storage (SMS)
+	Service 5 - Advice of Charge (AoC)
+	Service 6 - Capability Configuration Parameters (CCP)
+	Service 7 - PLMN selector
+	Service 8 - RFU
+	Service 9 - MSISDN
+	Service 10 - Extension1
+	Service 13 - Last Number Dialled (LND)
+	Service 14 - Cell Broadcast Message Identifier
+	Service 17 - Service Provider Name
+	Service 18 - Service Dialling Numbers (SDN)
+	Service 19 - Extension3
+	Service 20 - RFU
+	Service 21 - VGCS Group Identifier List (EFVGCS and EFVGCSS)
+	Service 22 - VBS Group Identifier List (EFVBS and EFVBSS)
+	Service 23 - enhanced Multi-Level Precedence and Pre-emption Service
+	Service 24 - Automatic Answer for eMLPP
+	Service 25 - Data download via SMS-CB
+	Service 26 - Data download via SMS-PP
+	Service 27 - Menu selection
+	Service 28 - Call control
+	Service 29 - Proactive SIM
+	Service 30 - Cell Broadcast Message Identifier Ranges
+	Service 31 - Barred Dialling Numbers (BDN)
+	Service 32 - Extension4
+	Service 33 - De-personalization Control Keys
+	Service 34 - Co-operative Network List
+	Service 35 - Short Message Status Reports
+	Service 36 - Network's indication of alerting in the MS
+	Service 37 - Mobile Originated Short Message control by SIM
+	Service 38 - GPRS
+	Service 49 - MExE
+	Service 50 - Reserved and shall be ignored
+	Service 51 - PLMN Network Name
+	Service 52 - Operator PLMN List
+	Service 53 - Mailbox Dialling Numbers
+	Service 54 - Message Waiting Indication Status
+	Service 57 - Multimedia Messaging Service (MMS)
+	Service 58 - Extension 8
+	Service 59 - MMS User Connectivity Parameters
+
+EHPLMN:
+	00f110 # MCC: 001 MNC: 001
+	ffffff # unused
+	ffffff # unused
+	ffffff # unused
+
+USIM Service Table: beff9f9de73e0408400170330000002e00000000
+	Service 2 - Fixed Dialling Numbers (FDN)
+	Service 3 - Extension 2
+	Service 4 - Service Dialling Numbers (SDN)
+	Service 5 - Extension3
+	Service 6 - Barred Dialling Numbers (BDN)
+	Service 8 - Outgoing Call Information (OCI and OCT)
+	Service 9 - Incoming Call Information (ICI and ICT)
+	Service 10 - Short Message Storage (SMS)
+	Service 11 - Short Message Status Reports (SMSR)
+	Service 12 - Short Message Service Parameters (SMSP)
+	Service 13 - Advice of Charge (AoC)
+	Service 14 - Capability Configuration Parameters 2 (CCP2)
+	Service 15 - Cell Broadcast Message Identifier
+	Service 16 - Cell Broadcast Message Identifier Ranges
+	Service 17 - Group Identifier Level 1
+	Service 18 - Group Identifier Level 2
+	Service 19 - Service Provider Name
+	Service 20 - User controlled PLMN selector with Access Technology
+	Service 21 - MSISDN
+	Service 24 - Enhanced Multi-Level Precedence and Pre-emption Service
+	Service 25 - Automatic Answer for eMLPP
+	Service 27 - GSM Access
+	Service 28 - Data download via SMS-PP
+	Service 29 - Data download via SMS-CB
+	Service 32 - RUN AT COMMAND command
+	Service 33 - shall be set to 1
+	Service 34 - Enabled Services Table
+	Service 35 - APN Control List (ACL)
+	Service 38 - GSM security context
+	Service 39 - CPBCCH Information
+	Service 40 - Investigation Scan
+	Service 42 - Operator controlled PLMN selector with Access Technology
+	Service 43 - HPLMN selector with Access Technology
+	Service 44 - Extension 5
+	Service 45 - PLMN Network Name
+	Service 46 - Operator PLMN List
+	Service 51 - Service Provider Display Information
+	Service 60 - User Controlled PLMN selector for I-WLAN access
+	Service 71 - Equivalent HPLMN
+	Service 73 - Equivalent HPLMN Presentation Indication
+	Service 85 - EPS Mobility Management Information
+	Service 86 - Allowed CSG Lists and corresponding indications
+	Service 87 - Call control on EPS PDN connection by USIM
+	Service 89 - eCall Data
+	Service 90 - Operator CSG Lists and corresponding indications
+	Service 93 - Communication Control for IMS by USIM
+	Service 94 - Extended Terminal Applications
+	Service 122 - 5GS Mobility Management Information
+	Service 123 - 5G Security Parameters
+	Service 124 - Subscription identifier privacy support
+	Service 126 - UAC Access Identities support
+
+ePDGId:
+	Not available
+
+ePDGSelection:
+	ffffffffffff # unused
+	ffffffffffff # unused
+	ffffffffffff # unused
+	ffffffffffff # unused
+
+P-CSCF:
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+
+Home Network Domain Name: Not available
+IMS private user identity: Not available
+IMS public user identity:
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+
+UICC IARI:
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+	Not available
+
+ISIM Service Table: 190200
+	Service 1 - P-CSCF address
+	Service 4 - GBA-based Local Key Establishment Mechanism
+	Service 5 - Support of P-CSCF discovery for IMS Local Break Out
+	Service 10 - Support of UICC access to IMS
+
+Done !
+

pysim-testdata/sysmoUSIM-SJS1.data

@@ -0,0 +1,8 @@
+MCC=001
+MNC=01
+ICCID=1122334455667788990
+KI=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+OPC=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+IMSI=001010000000102
+MSISDN=+77776336143
+ADM=55538407

pysim-testdata/sysmoUSIM-SJS1.ok

@@ -0,0 +1,144 @@
+Using PC/SC reader interface
+Reading ...
+Autodetected card type: sysmoUSIM-SJS1
+ICCID: 1122334455667788990
+IMSI: 001010000000102
+GID1: ffffffffffffffffffff
+GID2: ffffffffffffffffffff
+SMSP: ffffffffffffffffffffffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SPN: Magic
+Show in HPLMN: True
+Hide in OPLMN: True
+PLMNsel: 00f110ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+PLMNwAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+OPLMNwAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+HPLMNAcT:
+	00f110ffff # MCC: 001 MNC: 01 AcT: UTRAN, E-UTRAN WB-S1, E-UTRAN NB-S1, NG-RAN, GSM, GSM COMPACT, cdma2000 HRPD, cdma2000 1xRTT
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+	ffffff0000 # unused
+
+ACC: 0008
+MSISDN (NPI=1 ToN=1): +77776336143
+Administrative data: 00000002
+	MS operation mode: normal
+	Ciphering Indicator: disabled
+SIM Service Table: ff3fffff3f003f1ff00c00c0f00000
+	Service 1 - CHV1 disable function
+	Service 2 - Abbreviated Dialling Numbers (ADN)
+	Service 3 - Fixed Dialling Numbers (FDN)
+	Service 4 - Short Message Storage (SMS)
+	Service 5 - Advice of Charge (AoC)
+	Service 6 - Capability Configuration Parameters (CCP)
+	Service 7 - PLMN selector
+	Service 8 - RFU
+	Service 9 - MSISDN
+	Service 10 - Extension1
+	Service 11 - Extension2
+	Service 12 - SMS Parameters
+	Service 13 - Last Number Dialled (LND)
+	Service 14 - Cell Broadcast Message Identifier
+	Service 17 - Service Provider Name
+	Service 18 - Service Dialling Numbers (SDN)
+	Service 19 - Extension3
+	Service 20 - RFU
+	Service 21 - VGCS Group Identifier List (EFVGCS and EFVGCSS)
+	Service 22 - VBS Group Identifier List (EFVBS and EFVBSS)
+	Service 23 - enhanced Multi-Level Precedence and Pre-emption Service
+	Service 24 - Automatic Answer for eMLPP
+	Service 25 - Data download via SMS-CB
+	Service 26 - Data download via SMS-PP
+	Service 27 - Menu selection
+	Service 28 - Call control
+	Service 29 - Proactive SIM
+	Service 30 - Cell Broadcast Message Identifier Ranges
+	Service 31 - Barred Dialling Numbers (BDN)
+	Service 32 - Extension4
+	Service 33 - De-personalization Control Keys
+	Service 34 - Co-operative Network List
+	Service 35 - Short Message Status Reports
+	Service 36 - Network's indication of alerting in the MS
+	Service 37 - Mobile Originated Short Message control by SIM
+	Service 38 - GPRS
+	Service 49 - MExE
+	Service 50 - Reserved and shall be ignored
+	Service 51 - PLMN Network Name
+	Service 52 - Operator PLMN List
+	Service 53 - Mailbox Dialling Numbers
+	Service 54 - Message Waiting Indication Status
+	Service 57 - Multimedia Messaging Service (MMS)
+	Service 58 - Extension 8
+	Service 59 - MMS User Connectivity Parameters
+
+USIM Service Table: 9e6b1dfc67f6580000
+	Service 2 - Fixed Dialling Numbers (FDN)
+	Service 3 - Extension 2
+	Service 4 - Service Dialling Numbers (SDN)
+	Service 5 - Extension3
+	Service 8 - Outgoing Call Information (OCI and OCT)
+	Service 9 - Incoming Call Information (ICI and ICT)
+	Service 10 - Short Message Storage (SMS)
+	Service 12 - Short Message Service Parameters (SMSP)
+	Service 14 - Capability Configuration Parameters 2 (CCP2)
+	Service 15 - Cell Broadcast Message Identifier
+	Service 17 - Group Identifier Level 1
+	Service 19 - Service Provider Name
+	Service 20 - User controlled PLMN selector with Access Technology
+	Service 21 - MSISDN
+	Service 27 - GSM Access
+	Service 28 - Data download via SMS-PP
+	Service 29 - Data download via SMS-CB
+	Service 30 - Call Control by USIM
+	Service 31 - MO-SMS Control by USIM
+	Service 32 - RUN AT COMMAND command
+	Service 33 - shall be set to 1
+	Service 34 - Enabled Services Table
+	Service 35 - APN Control List (ACL)
+	Service 38 - GSM security context
+	Service 39 - CPBCCH Information
+	Service 42 - Operator controlled PLMN selector with Access Technology
+	Service 43 - HPLMN selector with Access Technology
+	Service 45 - PLMN Network Name
+	Service 46 - Operator PLMN List
+	Service 47 - Mailbox Dialling Numbers
+	Service 48 - Message Waiting Indication Status
+	Service 52 - Multimedia Messaging Service (MMS)
+	Service 53 - Extension 8
+	Service 55 - MMS User Connectivity Parameters
+
+Done !
+

pysim-testdata/sysmosim-gr1.data

@@ -0,0 +1,7 @@
+MCC=001
+MNC=01
+ICCID=1122334455667788990
+KI=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+OPC=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+IMSI=001010000000102
+ADM=DDDDDDDD

pysim-testdata/sysmosim-gr1.ok

@@ -0,0 +1,57 @@
+Using PC/SC reader interface
+Reading ...
+Autodetected card type: sysmosim-gr1
+Can't read AIDs from SIM -- SW match failed! Expected 9000 and got 9404.
+ICCID: 1122334455667788990
+IMSI: 001010000000102
+GID1: Can't read file -- SW match failed! Expected 9000 and got 9404.
+GID2: Can't read file -- SW match failed! Expected 9000 and got 9404.
+SMSP: ffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SPN: Not available
+Show in HPLMN: False
+Hide in OPLMN: False
+PLMNsel: 00f110ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+PLMNwAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
+OPLMNwAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
+HPLMNAcT: Can't read file -- SW match failed! Expected 9000 and got 9404.
+ACC: 0008
+MSISDN: Not available
+Administrative data: 000000
+	MS operation mode: normal
+	Ciphering Indicator: disabled
+SIM Service Table: ff3fff0f0f0000030000
+	Service 1 - CHV1 disable function
+	Service 2 - Abbreviated Dialling Numbers (ADN)
+	Service 3 - Fixed Dialling Numbers (FDN)
+	Service 4 - Short Message Storage (SMS)
+	Service 5 - Advice of Charge (AoC)
+	Service 6 - Capability Configuration Parameters (CCP)
+	Service 7 - PLMN selector
+	Service 8 - RFU
+	Service 9 - MSISDN
+	Service 10 - Extension1
+	Service 11 - Extension2
+	Service 12 - SMS Parameters
+	Service 13 - Last Number Dialled (LND)
+	Service 14 - Cell Broadcast Message Identifier
+	Service 17 - Service Provider Name
+	Service 18 - Service Dialling Numbers (SDN)
+	Service 19 - Extension3
+	Service 20 - RFU
+	Service 21 - VGCS Group Identifier List (EFVGCS and EFVGCSS)
+	Service 22 - VBS Group Identifier List (EFVBS and EFVBSS)
+	Service 23 - enhanced Multi-Level Precedence and Pre-emption Service
+	Service 24 - Automatic Answer for eMLPP
+	Service 25 - Data download via SMS-CB
+	Service 26 - Data download via SMS-PP
+	Service 27 - Menu selection
+	Service 28 - Call control
+	Service 33 - De-personalization Control Keys
+	Service 34 - Co-operative Network List
+	Service 35 - Short Message Status Reports
+	Service 36 - Network's indication of alerting in the MS
+	Service 57 - Multimedia Messaging Service (MMS)
+	Service 58 - Extension 8
+
+Done !
+

requirements.txt

@@ -0,0 +1,13 @@
+pyscard
+pyserial
+pytlv
+cmd2==1.5
+jsonpath-ng
+construct
+bidict
+gsm0338
+pyyaml>=5.1
+termcolor
+colorlog
+pycryptodome
+git+https://github.com/hologram-io/smpp.pdu

scripts/deactivate-5g.script

@@ -0,0 +1,70 @@
+# script to be used with pySim-shell.py which is part of the Osmocom pysim package,
+# found at https://osmocom.org/projects/pysim/wiki
+set echo true
+
+# this script will deactivate all 5G related services and files.  This can be used
+# in case you do not wish to use any 5G services, or you do not wish to configure
+# the 5G specific files on the USIM card.  The card will then behave like a 3G USIM
+# without any 5G capability, using the default fall-back mechanisms specified by 3GPP.
+
+# TODO: add your card-specific ADM pin at the end of the verify_adm line below
+verify_adm
+
+# deactivate any 5G related services in EF.UST
+select ADF.USIM
+select EF.UST
+ust_service_deactivate 122
+ust_service_deactivate 123
+ust_service_deactivate 124
+ust_service_deactivate 125
+ust_service_deactivate 126
+ust_service_deactivate 127
+ust_service_deactivate 129
+ust_service_deactivate 130
+ust_service_deactivate 132
+ust_service_deactivate 133
+ust_service_deactivate 134
+ust_service_deactivate 135
+
+# deactivate all files in EF.5GS
+select ADF.USIM
+select DF.5GS
+
+select EF.5GAUTHKEYS
+deactivate_file
+
+select EF.5GS3GPPLOCI
+deactivate_file
+
+select EF.5GSN3GPPNSC
+deactivate_file
+
+select EF.5GSN3GPPLOCI
+deactivate_file
+
+select EF.5GS3GPPNSC
+deactivate_file
+
+# only exists on sysmoISIM-SJA2v2
+select EF.OPL5G
+deactivate_file
+
+select EF.Routing_Indicator
+deactivate_file
+
+select EF.SUCI_Calc_Info
+deactivate_file
+
+select EF.SUPI_NAI
+deactivate_file
+
+# only exists on sysmoISIM-SJA2v2
+select EF.TN3GPPSNN
+deactivate_file
+
+select EF.UAC_AIC
+deactivate_file
+
+# only exists on sysmoISIM-SJA2v2
+select EF.URSP
+deactivate_file

scripts/deactivate-ims.script

@@ -0,0 +1,74 @@
+# script to be used with pySim-shell.py which is part of the Osmocom pysim package,
+# found at https://osmocom.org/projects/pysim/wiki
+set echo true
+
+# this script will deactivate all IMS related services and files.  This can be used
+# in case you do not wish to use any IMS services, or you do not wish to configure
+# the IMS specific files on the USIM/ISIM cards.  The card will then behave like a 3G USIM
+# without any IMS capability, using the default fall-back mechanisms specified by 3GPP.
+
+# TODO: add your card-specific ADM pin at the end of the verify_adm line below
+verify_adm
+
+# deactivate any IMS related services in EF.UST
+select ADF.USIM
+select EF.UST
+ust_service_deactivate 93
+ust_service_deactivate 95
+ust_service_deactivate 104
+ust_service_deactivate 105
+ust_service_deactivate 106
+ust_service_deactivate 107
+ust_service_deactivate 108
+ust_service_deactivate 109
+ust_service_deactivate 110
+ust_service_deactivate 112
+ust_service_deactivate 114
+ust_service_deactivate 115
+ust_service_deactivate 118
+ust_service_deactivate 120
+ust_service_deactivate 131
+ust_service_deactivate 134
+
+# deactivate all IMS related files in ADF.USIM
+select ADF.USIM
+
+select EF.UICCIARI
+deactivate_file
+
+select EF.ePDGId
+deactivate_file
+
+select EF.ePDGSelection
+deactivate_file
+
+select EF.ePDGIdEm
+deactivate_file
+
+select EF.ePDGSelectionEm
+deactivate_file
+
+select EF.FromPreferred
+deactivate_file
+
+select EF.IMSConfigData
+deactivate_file
+
+select EF.3GPPPSDATAOFF
+deactivate_file
+
+select EF.3GPPPSDATAOFFservicelist
+deactivate_file
+
+select EF.XCAPConfigData
+deactivate_file
+
+select EF.MuDMiDConfigData
+deactivate_file
+
+echo "Please make sure to manually disable the ISIM applet as described in the end of the script"
+# you can currently only manually do this via GlobalPlatformPro or some other tool using
+# java -jar ./gp.jar --key-enc KIC1 --key-mac KID1 --key-dek KIK1 --lock-applet A0000000871004FFFFFFFF8907090000
+# (substituting KIC1/KID1/KIK1 with the card-specific keys, of course)
+
+quit

scripts/sysmoISIM-SJA2/dump-auth-cfg.pysim

@@ -0,0 +1,41 @@
+# script to be used with pySim-shell.py which is part of the Osmocom pysim package,
+# found at https://osmocom.org/projects/pysim/wiki
+set echo true
+
+# TODO: add your card-specific ADM pin at the end of the verify_adm line below
+verify_adm
+
+select DF.SYSTEM
+
+# Milenage configuration (constants)
+select EF.MILENAGE_CFG
+read_binary_decoded
+
+# 2G authentication kay / algorithm
+select EF.SIM_AUTH_KEY
+read_binary_decoded
+
+# OTA keys
+#select EF.0348_KEY
+#read_records_decoded
+
+select ADF.USIM
+# USIM authentication key / algoritmh in 3G security context
+select EF.USIM_AUTH_KEY
+read_binary_decoded
+# USIM authentication key / algorithm in 2G security context
+select EF.USIM_AUTH_KEY_2G
+read_binary_decoded
+# USIM SQN numbers
+select EF.USIM_SQN
+read_binary_decoded
+
+select ADF.ISIM
+# ISIM authentication key / algorithm
+select EF.ISIM_AUTH_KEY
+read_binary_decoded
+# ISIM SQN numbers
+select EF.ISIM_SQN
+read_binary_decoded
+
+quit

setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+long_description = file: README.md
+long_description_content_type = text/markdown

setup.py

@@ -0,0 +1,30 @@
+from setuptools import setup
+
+setup(
+    name='pySim',
+    version='1.1',
+    packages=['pySim', 'pySim.transport'],
+    url='https://osmocom.org/projects/pysim/wiki',
+    license='GPLv2',
+    author_email='simtrace@lists.osmocom.org',
+    description='Tools related to SIM/USIM/ISIM cards',
+    install_requires=[
+        "pyscard",
+        "serial",
+        "pytlv",
+        "cmd2 >= 1.3.0, < 2.0.0",
+        "jsonpath-ng",
+        "construct >= 2.9",
+        "bidict",
+        "gsm0338",
+        "termcolor",
+        "colorlog",
+        "pycryptodome",
+        "smpp.pdu @ git+https://github.com/hologram-io/smpp.pdu",
+    ],
+    scripts=[
+        'pySim-prog.py',
+        'pySim-read.py',
+        'pySim-shell.py'
+    ]
+)

smpp2sim.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+#
+# Program to emulate the entire communication path SMSC-MSC-BSC-BTS-ME
+# that is usually between an OTA backend and the SIM card.  This allows
+# to play with SIM OTA technology without using a mobile network or even
+# a mobile phone.
+#
+# An external application must encode (and encrypt/sign) the OTA SMS
+# and submit them via SMPP to this program, just like it would submit
+# it normally to a SMSC (SMS Service Centre).  The program then re-formats
+# the SMPP-SUBMIT into a SMS DELIVER TPDU and passes it via an ENVELOPE
+# APDU to the SIM card that is locally inserted into a smart card reader.
+#
+# The path from SIM to external OTA application works the opposite way.
+
+import argparse
+import logging
+import colorlog
+from pprint import pprint as pp
+
+from twisted.protocols import basic
+from twisted.internet import defer, endpoints, protocol, reactor, task
+from twisted.cred.portal import IRealm
+from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
+from twisted.cred.portal import Portal
+from zope.interface import implementer
+
+from smpp.twisted.config import SMPPServerConfig
+from smpp.twisted.server import SMPPServerFactory, SMPPBindManager
+from smpp.twisted.protocol import SMPPSessionStates, DataHandlerResponse
+
+from smpp.pdu import pdu_types, operations, pdu_encoding
+
+from pySim.sms import SMS_DELIVER, AddressField
+
+from pySim.transport import LinkBase, ProactiveHandler, argparse_add_reader_args, init_reader
+from pySim.commands import SimCardCommands
+from pySim.cards import UsimCard
+from pySim.exceptions import *
+from pySim.cat import ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload
+from pySim.cat import DeviceIdentities, Address
+from pySim.utils import b2h, h2b
+
+logger = logging.getLogger(__name__)
+
+# MSISDNs to use when generating proactive SMS messages
+SIM_MSISDN='23'
+ESME_MSISDN='12'
+
+# HACK: we need some kind of mapping table between system_id and card-reader
+# or actually route based on MSISDNs
+hackish_global_smpp = None
+
+class Proact(ProactiveHandler):
+    def __init__(self, smpp_factory):
+        self.smpp_factory = smpp_factory
+
+    @staticmethod
+    def _find_first_element_of_type(instlist, cls):
+        for i in instlist:
+            if isinstance(i, cls):
+                return i
+        return None
+
+    """Call-back which the pySim transport core calls whenever it receives a
+    proactive command from the SIM."""
+    def handle_SendShortMessage(self, data):
+        """Card requests sending a SMS."""
+        pp(data)
+        # Relevant parts in data: Address, SMS_TPDU
+        addr_ie = _find_first_element_of_type(data.children, Address)
+        sms_tpdu_ie = _find_first_element_of_type(data.children, SMS_TPDU)
+        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
+        submit = SMS_SUBMIT.fromBytes(raw_tpdu)
+        self.send_sms_via_smpp(data)
+    def handle_OpenChannel(self, data):
+        """Card requests opening a new channel via a UDP/TCP socket."""
+        pp(data)
+        pass
+    def handle_CloseChannel(self, data):
+        """Close a channel."""
+        pp(data)
+        pass
+    def handleReceiveData(self, data):
+        """Receive/read data from the socket."""
+        pp(data)
+        pass
+    def handleSendData(self, data):
+        """Send/write data to the socket."""
+        pp(data)
+        pass
+    def getChannelStatus(self, data):
+        pp(data)
+        pass
+
+    def send_sms_via_smpp(self, data):
+        # while in a normal network the phone/ME would *submit* a message to the SMSC,
+        # we are actually emulating the SMSC itself, so we must *deliver* the message
+        # to the ESME
+        dcs = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT,
+                                       gsmFeatures=[pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET])
+        deliver = operations.DeliverSM(source_addr=SIM_MSISDN,
+                                       destination_addr=ESME_MSISDN,
+                                       esm_class=esm_class,
+                                       protocol_id=0x7F,
+                                       data_coding=dcs,
+                                       short_message=h2b(data))
+        hackish_global_smpp.sendDataRequest(deliver)
+#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
+#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
+#       connection.sendDataRequest(deliver)
+
+
+
+def dcs_is_8bit(dcs):
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
+        return True
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED_COMMON):
+        return True
+    if dcs.scheme == pdu_types.DataCodingScheme.GSM_MESSAGE_CLASS and dcs.schemeData['msgCoding'] == pdu_types.DataCodingGsmMsgCoding.DATA_8BIT:
+        return True
+    else:
+        return False
+
+
+class MyServer:
+
+    @implementer(IRealm)
+    class SmppRealm:
+        def requestAvatar(self, avatarId, mind, *interfaces):
+            return ('SMPP', avatarId, lambda: None)
+
+    def __init__(self, tcp_port:int = 2775, bind_ip = '::'):
+        smpp_config = SMPPServerConfig(msgHandler=self._msgHandler,
+                                       systems={'test': {'max_bindings': 2}})
+        portal = Portal(self.SmppRealm())
+        credential_checker = InMemoryUsernamePasswordDatabaseDontUse()
+        credential_checker.addUser('test', 'test')
+        portal.registerChecker(credential_checker)
+        self.factory = SMPPServerFactory(smpp_config, auth_portal=portal)
+        logger.info('Binding Virtual SMSC to TCP Port %u at %s' % (tcp_port, bind_ip))
+        smppEndpoint = endpoints.TCP6ServerEndpoint(reactor, tcp_port, interface=bind_ip)
+        smppEndpoint.listen(self.factory)
+        self.tp = self.scc = self.card = None
+
+    def connect_to_card(self, tp: LinkBase):
+        self.tp = tp
+        self.scc = SimCardCommands(self.tp)
+        self.card = UsimCard(self.scc)
+        # this should be part of UsimCard, but FairewavesSIM breaks with that :/
+        self.scc.cla_byte = "00"
+        self.scc.sel_ctrl = "0004"
+        self.card.read_aids()
+        self.card.select_adf_by_aid(adf='usim')
+        # FIXME: create a more realistic profile than ffffff
+        self.scc.terminal_profile('ffffff')
+
+    def _msgHandler(self, system_id, smpp, pdu):
+        # HACK: we need some kind of mapping table between system_id and card-reader
+        # or actually route based on MSISDNs
+        global hackish_global_smpp
+        hackish_global_smpp = smpp
+        #pp(pdu)
+        if pdu.id == pdu_types.CommandId.submit_sm:
+            return self.handle_submit_sm(system_id, smpp, pdu)
+        else:
+            logging.warning('Rejecting non-SUBMIT commandID')
+            return pdu_types.CommandStatus.ESME_RINVCMDID
+
+    def handle_submit_sm(self, system_id, smpp, pdu):
+        # check for valid data coding scheme + PID
+        if not dcs_is_8bit(pdu.params['data_coding']):
+            logging.warning('Rejecting non-8bit DCS')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+        if pdu.params['protocol_id'] != 0x7f:
+            logging.warning('Rejecting non-SIM PID')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+
+        # 1) build a SMS-DELIVER (!) from the SMPP-SUBMIT
+        tpdu = SMS_DELIVER.fromSmppSubmit(pdu)
+        print(tpdu)
+        # 2) wrap into the CAT ENVELOPE for SMS-PP-Download
+        tpdu_ie = SMS_TPDU(decoded={'tpdu': b2h(tpdu.toBytes())})
+        dev_ids = DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'})
+        sms_dl = SMSPPDownload(children=[dev_ids, tpdu_ie])
+        # 3) send to the card
+        envelope_hex = b2h(sms_dl.to_tlv())
+        print("ENVELOPE: %s" % envelope_hex)
+        (data, sw) = self.scc.envelope(envelope_hex)
+        print("SW %s: %s" % (sw, data))
+        if sw == '9300':
+            # TODO send back RP-ERROR message with TP-FCS == 'SIM Application Toolkit Busy'
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+        elif sw == '9000' or sw[0:2] in ['6f', '62', '63']:
+            # data something like 027100000e0ab000110000000000000001612f or
+            # 027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c
+            # which is the user-data portion of the SMS starting with the UDH (027100)
+            # TODO: return the response back to the sender in an RP-ACK; PID/DCS like in CMD
+            deliver = operations.DeliverSM(service_type=pdu.params['service_type'],
+                                           source_addr_ton=pdu.params['dest_addr_ton'],
+                                           source_addr_npi=pdu.params['dest_addr_npi'],
+                                           source_addr=pdu.params['destination_addr'],
+                                           dest_addr_ton=pdu.params['source_addr_ton'],
+                                           dest_addr_npi=pdu.params['source_addr_npi'],
+                                           destination_addr=pdu.params['source_addr'],
+                                           esm_class=pdu.params['esm_class'],
+                                           protocol_id=pdu.params['protocol_id'],
+                                           priority_flag=pdu.params['priority_flag'],
+                                           data_coding=pdu.params['data_coding'],
+                                           short_message=h2b(data))
+            smpp.sendDataRequest(deliver)
+            return pdu_types.CommandStatus.ESME_ROK
+        else:
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+
+
+option_parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+argparse_add_reader_args(option_parser)
+smpp_group = option_parser.add_argument_group('SMPP Options')
+smpp_group.add_argument('--smpp-bind-port', type=int, default=2775,
+                        help='TCP Port to bind the SMPP socket to')
+smpp_group.add_argument('--smpp-bind-ip', default='::',
+                        help='IPv4/IPv6 address to bind the SMPP socket to')
+
+if __name__ == '__main__':
+    log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+    colorlog.basicConfig(level=logging.INFO, format = log_format)
+    logger = colorlog.getLogger()
+
+    opts = option_parser.parse_args()
+
+    #tp = init_reader(opts, proactive_handler = Proact())
+    tp = init_reader(opts)
+    if tp is None:
+        exit(1)
+    tp.connect()
+
+    ms = MyServer(opts.smpp_bind_port, opts.smpp_bind_ip)
+    ms.connect_to_card(tp)
+    reactor.run()
+

sms_test.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+from pySim.sms import *
+from pprint import pprint as pp
+from construct import setGlobalPrintPrivateEntries
+
+
+print(UserDataHeader.fromBytes('027100'))
+print(UserDataHeader.fromBytes('027100abcdef'))
+print(UserDataHeader.fromBytes('03710110'))
+print(UserDataHeader.fromBytes('0571007001ffabcd'))
+
+setGlobalPrintPrivateEntries(True)
+pp(AddressField.fromBytes('0480214399'))
+
+s = SMS_SUBMIT.fromBytes('550d0b911614261771f000f5a78c0b050423f423f40003010201424547494e3a56434152440d0a56455253494f4e3a322e310d0a4e3a4d650d0a54454c3b505245463b43454c4c3b564f4943453a2b36313431363237313137300d0a54454c3b484f4d453b564f4943453a2b36313339353337303437310d0a54454c3b574f524b3b564f4943453a2b36313339363734373031350d0a454e443a')
+pp(s)
+print(s.tp_da)
+pp(b2h(s.toBytes()))
+
+d = SMS_DELIVER.fromBytes('0408D0E5759A0E7FF6907090307513000824010101BB400101')
+pp(d)
+pp(b2h(d.toBytes()))

tests/pysim-test.sh

@@ -0,0 +1,230 @@
+#!/bin/bash
+
+# Utility to verify the functionality of pysim-prog.py
+#
+# (C) 2018 by Sysmocom s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Philipp Maier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+PYSIM_PROG=../pySim-prog.py
+PYSIM_READ=../pySim-read.py
+TEMPFILE=temp.tmp
+PYTHON=python3
+
+set -e
+
+echo "pysim-test - a test program to test pysim-prog.py"
+echo "================================================="
+
+# Generate a list of the cards we expect to see by checking which .ok files
+# are present
+function gen_card_list {
+    N_CARDS=0
+
+    echo "Expecting to see the following cards:"
+
+    for I in *.data ; do
+	CARD_NAMES[$N_CARDS]=${I%.*}
+	CARD_SEEN[$N_CARDS]=0
+	N_CARDS=$((N_CARDS+1))
+    done
+
+    for I in $(seq 0 $((N_CARDS-1))); do
+	echo ${CARD_NAMES[$I]}
+    done
+}
+
+# Increment counter in card list for a specified card name (type)
+function inc_card_list {
+    CARD_NAME=$1
+    for I in $(seq 0 $((N_CARDS-1))); do
+	if [ $CARD_NAME = ${CARD_NAMES[$I]} ]; then
+	    CARD_SEEN[$I]=$((${CARD_NAMES[$I]}+1))
+	fi
+    done
+}
+
+# Check the card list, each card must be seen exactly one times
+function check_card_list {
+    for I in $(seq 0 $((N_CARDS-1))); do
+	if [ ${CARD_SEEN[$I]} -ne 1 ]; then
+	    echo "Error: Card ${CARD_NAMES[$I]} seen ${CARD_SEEN[$I]} times!"
+	    exit 1
+	fi
+    done
+
+    echo "All cards seen -- everything ok!"
+}
+
+# Verify the contents of a card by reading them and then diffing against the
+# previously created .ok file
+function check_card {
+    TERMINAL=$1
+    CARD_NAME=$2
+    echo "Verifying card ..."
+    stat ./$CARD_NAME.ok > /dev/null
+    $PYTHON $PYSIM_READ -p $TERMINAL > $TEMPFILE
+    set +e
+    CARD_DIFF=$(diff $TEMPFILE ./$CARD_NAME.ok)
+    set -e
+
+    if [ "$CARD_DIFF" != "" ]; then
+	echo "Card contents do not match the test data:"
+	echo "Expected: $CARD_NAME.ok"
+	echo "------------8<------------"
+	cat "$CARD_NAME.ok"
+	echo "------------8<------------"
+	echo "Got:"
+	echo "------------8<------------"
+	cat $TEMPFILE
+	echo "------------8<------------"
+	rm *.tmp
+	exit 1
+    fi
+
+    inc_card_list $CARD_NAME
+
+    echo "Card contents match the test data -- success!"
+    rm $TEMPFILE
+}
+
+# Read out the card using pysim-read and store the result as .ok file. This
+# data will be used later in order to verify the results of our write tests.
+function gen_ok_file {
+    TERMINAL=$1
+    CARD_NAME=$2
+    $PYTHON $PYSIM_READ -p $TERMINAL > "$CARD_NAME.ok"
+    echo "Generated file: $CARD_NAME.ok"
+    echo "------------8<------------"
+    cat "$CARD_NAME.ok"
+    echo "------------8<------------"
+}
+
+# Find out the type (card name) of the card that is installed in the specified
+# reader
+function probe_card {
+    TERMINAL=$1
+    RESULT=$(timeout 5 $PYSIM_PROG -p $TERMINAL -T | cut -d ":" -f 2 | tail -n 1 | xargs)
+    echo $RESULT
+}
+
+# Read out all cards and store the results as .ok files
+function gen_ok_files {
+    echo "== OK FILE GENERATION =="
+    for I in $(seq 0 $((N_TERMINALS-1))); do
+	echo "Probing card in terminal #$I"
+	CARD_NAME=$(probe_card $I)
+	if [ -z "$CARD_NAME" ]; then
+	    echo "Error: Unresponsive card!"
+	    exit 1
+	fi
+	echo "Card is of type: $CARD_NAME"
+	gen_ok_file $I $CARD_NAME
+    done
+}
+
+# Execute tests. Each card is programmed and the contents are checked
+# afterwards.
+function run_test {
+    for I in $(seq 0 $((N_TERMINALS-1))); do
+	echo "== EXECUTING TEST =="
+	echo "Probing card in terminal #$I"
+	CARD_NAME=$(probe_card $I)
+	if [ -z "$CARD_NAME" ]; then
+	    echo "Error: Unresponsive card!"
+	    exit 1
+	fi
+	echo "Card is of type: $CARD_NAME"
+
+	# Make sure some default data is set
+	MCC=001
+	MNC=01
+	ICCID=1122334455667788990
+	KI=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+	OPC=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+	IMSI=001010000000001
+	MSISDN=6766266
+	ADM=00000000
+	ADM_HEX=""
+	ADM_OPT="-a"
+
+	source "$CARD_NAME.data"
+	if [ -n "$ADM_HEX" ]; then
+		ADM_OPT="-A"
+		ADM=$ADM_HEX
+	fi
+	$PYTHON $PYSIM_PROG -p $I -t $CARD_NAME -o $OPC -k $KI -x $MCC -y $MNC -i $IMSI -s $ICCID --msisdn $MSISDN $ADM_OPT $ADM
+	check_card $I $CARD_NAME
+	echo ""
+    done
+}
+
+function usage {
+    echo "Options:"
+    echo "-n: number of card terminals"
+    echo "-o: generate .ok files"
+}
+
+# Make sure that the pathes to the python scripts always work, regardless from
+# where the script is called.
+CURDIR=$PWD
+SCRIPTDIR=$(dirname $0)
+cd $SCRIPTDIR
+PYSIM_PROG=$(realpath $PYSIM_PROG)
+PYSIM_READ=$(realpath $PYSIM_READ)
+cd $CURDIR
+
+OPT_N_TERMINALS=0
+OPT_GEN_OK_FILES=0
+while getopts ":hon:" OPT; do
+  case $OPT in
+      h)
+	  usage
+	  exit 0
+	  ;;
+      o)
+	  OPT_GEN_OK_FILES=1
+	  ;;
+      n)
+	  OPT_N_TERMINALS=$OPTARG
+	  ;;
+      \?)
+	  echo "Invalid option: -$OPTARG" >&2
+	  exit 1
+      ;;
+  esac
+done
+
+N_TERMINALS=$OPT_N_TERMINALS
+
+# Generate a list of available cards, if no explicit reader number is given
+# then the number of cards will be used as reader number.
+gen_card_list
+if [ $N_TERMINALS -eq 0 ]; then
+    N_TERMINALS=$N_CARDS
+fi
+echo "Number of card terminals installed: $N_TERMINALS"
+echo ""
+
+if [ $OPT_GEN_OK_FILES -eq 1 ]; then
+    gen_ok_files
+    exit 0
+else
+    run_test
+    check_card_list
+    exit 0
+fi

tests/test_apdu.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.utils import h2b, b2h
+from pySim.construct import filter_dict
+from pySim.apdu import Apdu
+from pySim.apdu.ts_31_102 import UsimAuthenticateEven
+
+class TestApdu(unittest.TestCase):
+    def test_successful(self):
+        apdu = Apdu('00a40400023f00', '9000')
+        self.assertEqual(apdu.successful, True)
+        apdu = Apdu('00a40400023f00', '6733')
+        self.assertEqual(apdu.successful, False)
+
+    def test_successful_method(self):
+        """Test overloading of the success property with a custom method."""
+        class SwApdu(Apdu):
+            def _is_success(self):
+                return False
+        apdu = SwApdu('00a40400023f00', '9000')
+        self.assertEqual(apdu.successful, False)
+
+# TODO: Tests for TS 102 221 / 31.102 ApduCommands
+
+class TestUsimAuth(unittest.TestCase):
+    """Test decoding of the rather complex USIM AUTHENTICATE command."""
+    def test_2g(self):
+        apdu = ('80880080' + '09' + '080001020304050607',
+                '04a0a1a2a308b0b1b2b3b4b5b6b79000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'gsm'},
+                    'body': {'rand': '0001020304050607', 'autn': None}},
+            'rsp': {'body': {'sres': 'a0a1a2a3', 'kc': 'b0b1b2b3b4b5b6b7'}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_3g(self):
+        apdu = ('80880081' + '12' + '080001020304050607081011121314151617',
+                'DB' + '08' + 'a0a1a2a3a4a5a6a7' +
+                       '10' + 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf' +
+                       '10' + 'c0c1c2c3c4c5c6c7c8c9cacbcccdcecf' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'umts'},
+                    'body': {'rand': '0001020304050607', 'autn': '1011121314151617'}},
+            'rsp': {'body': {'tag': 219,
+                             'body': {
+                                 'res': 'a0a1a2a3a4a5a6a7',
+                                 'ck': 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf',
+                                 'ik': 'c0c1c2c3c4c5c6c7c8c9cacbcccdcecf',
+                                 'kc': None
+                                 }
+                             }
+                    }
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_3g_sync(self):
+        apdu = ('80880081' + '12' + '080001020304050607081011121314151617',
+                'DC' + '08' + 'a0a1a2a3a4a5a6a7' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'umts'},
+                    'body': {'rand': '0001020304050607', 'autn': '1011121314151617'}},
+            'rsp': {'body': {'tag': 220, 'body': {'auts': 'a0a1a2a3a4a5a6a7' }}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_vgcs(self):
+        apdu = ('80880082' + '0E' + '04' + '00010203' +
+                             '01' + '10' +
+                             '08' + '2021222324252627',
+                'DB' + '10' + 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'vgcs_vbs'},
+                    'body': { 'vk_id': '10', 'vservice_id': '00010203', 'vstk_rand': '2021222324252627'}},
+            'rsp': {'body': {'vstk': 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf'}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_construct.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.construct import *
+
+tests = [
+        ( b'\x80', 0x80 ),
+        ( b'\x80\x01', 0x8001 ),
+        ( b'\x80\x00\x01', 0x800001 ),
+        ( b'\x80\x23\x42\x01', 0x80234201 ),
+        ]
+
+class TestGreedyInt(unittest.TestCase):
+    def test_GreedyInt_decoder(self):
+        gi = GreedyInteger()
+        for t in tests:
+            self.assertEqual(gi.parse(t[0]), t[1])
+    def test_GreedyInt_encoder(self):
+        gi = GreedyInteger()
+        for t in tests:
+            self.assertEqual(t[0], gi.build(t[1]))
+        pass
+
+class TestUtils(unittest.TestCase):
+    def test_filter_dict(self):
+        inp = {'foo': 0xf00, '_bar' : 0xba5, 'baz': 0xba2 }
+        out = {'foo': 0xf00, 'baz': 0xba2 }
+        self.assertEqual(filter_dict(inp), out)
+
+    def test_filter_dict_nested(self):
+        inp = {'foo': 0xf00, 'nest': {'_bar' : 0xba5}, 'baz': 0xba2 }
+        out = {'foo': 0xf00, 'nest': {}, 'baz': 0xba2 }
+        self.assertEqual(filter_dict(inp), out)
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_ota.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.utils import h2b, b2h
+from pySim.ota import *
+
+class Test_SMS_3DES(unittest.TestCase):
+    tar = h2b('b00000')
+    """Test the OtaDialectSms for 3DES algorithms."""
+    def __init__(self, foo, **kwargs):
+        super().__init__(foo, **kwargs)
+        # KIC1 + KID1 of 8988211000000467285
+        KIC1 = h2b('D0FDA31990D8D64178601317191669B4')
+        KID1 = h2b('D24EB461799C5E035C77451FD9404463')
+        KIC3 = h2b('C21DD66ACAC13CB3BC8B331B24AFB57B')
+        KID3 = h2b('12110C78E678C25408233076AA033615')
+        self.od = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3, kic=KIC3,
+                            algo_auth='triple_des_cbc2', kid_idx=3, kid=KID3)
+        self.dialect = OtaDialectSms()
+        self.spi_base = {
+                'counter':'no_counter',
+                'ciphering': True,
+                'rc_cc_ds': 'cc',
+                'por_in_submit':False,
+                'por': 'por_required',
+                'por_shall_be_ciphered': True,
+                'por_rc_cc_ds': 'cc',
+            }
+
+    def _check_response(self, r):
+        self.assertEqual(r['number_of_commands'], 1)
+        self.assertEqual(r['last_status_word'], '612f')
+        self.assertEqual(r['last_response_data'], u'')
+        self.assertEqual(r['response_status'], 'por_ok')
+
+    def test_resp_3des_ciphered(self):
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = True
+        spi['por_rc_cc_ds'] = 'cc'
+        r = self.dialect.decode_resp(self.od, spi, '027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c')
+        self._check_response(r)
+
+    def test_resp_3des_signed(self):
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = False
+        spi['por_rc_cc_ds'] = 'cc'
+        r = self.dialect.decode_resp(self.od, spi, '027100001612b000110000000000000055f47118381175fb01612f')
+        self._check_response(r)
+
+    def test_resp_3des_signed_err(self):
+        """Expect an OtaCheckError exception if the computed CC != received CC"""
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = False
+        spi['por_rc_cc_ds'] = 'cc'
+        with self.assertRaises(OtaCheckError) as context:
+            r = self.dialect.decode_resp(self.od, spi, '027100001612b000110000000000000055f47118381175fb02612f')
+        self.assertTrue('!= Computed CC' in str(context.exception))
+
+    def test_resp_3des_none(self):
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = False
+        spi['por_rc_cc_ds'] = 'no_rc_cc_ds'
+        r = self.dialect.decode_resp(self.od, spi, '027100000e0ab000110000000000000001612f')
+        self._check_response(r)
+
+    def test_cmd_3des_ciphered(self):
+        spi = self.spi_base
+        spi['ciphering'] = True
+        spi['rc_cc_ds'] = 'no_rc_cc_ds'
+        r = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('00a40000023f00'))
+        self.assertEqual(b2h(r), '00180d04193535b000000c8478b552a4ffc5a8f099b83cad7123')
+
+    def test_cmd_3des_signed(self):
+        spi = self.spi_base
+        spi['ciphering'] = False
+        spi['rc_cc_ds'] = 'cc'
+        r = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('00a40000023f00'))
+        self.assertEqual(b2h(r), '1502193535b00000000000000000072ea17bdb72060e00a40000023f00')
+
+    def test_cmd_3des_none(self):
+        spi = self.spi_base
+        spi['ciphering'] = False
+        spi['rc_cc_ds'] = 'no_rc_cc_ds'
+        r = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('00a40000023f00'))
+        self.assertEqual(b2h(r), '0d00193535b0000000000000000000a40000023f00')

tests/test_sms.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.utils import h2b, b2h
+from pySim.sms import *
+
+class Test_SMS_UDH(unittest.TestCase):
+    def test_single_ie(self):
+        udh, tail = UserDataHeader.fromBytes('027100')
+        self.assertEqual(len(udh.ies), 1)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 0)
+        self.assertEqual(ie.value, b'')
+        self.assertEqual(tail, b'')
+
+    def test_single_ie_tail(self):
+        udh, tail = UserDataHeader.fromBytes('027100abcdef')
+        self.assertEqual(len(udh.ies), 1)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 0)
+        self.assertEqual(ie.value, b'')
+        self.assertEqual(tail, b'\xab\xcd\xef')
+
+    def test_single_ie_value(self):
+        udh, tail = UserDataHeader.fromBytes('03710110')
+        self.assertEqual(len(udh.ies), 1)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 1)
+        self.assertEqual(ie.value, b'\x10')
+        self.assertEqual(tail, b'')
+
+    def test_two_ie_data_tail(self):
+        udh, tail = UserDataHeader.fromBytes('0571007001ffabcd')
+        self.assertEqual(len(udh.ies), 2)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 0)
+        self.assertEqual(ie.value, b'')
+        ie = udh.ies[1]
+        self.assertEqual(ie.iei, 0x70)
+        self.assertEqual(ie.length, 1)
+        self.assertEqual(ie.value, b'\xff')
+        self.assertEqual(tail, b'\xab\xcd')
+
+    def test_toBytes(self):
+        indata = h2b('0571007001ff')
+        udh, tail = UserDataHeader.fromBytes(indata)
+        encoded = udh.toBytes()
+        self.assertEqual(encoded, indata)
+
+class Test_AddressField(unittest.TestCase):
+    def test_fromBytes(self):
+        encoded = h2b('0480214399')
+        af, trailer = AddressField.fromBytes(encoded)
+        self.assertEqual(trailer, b'\x99')
+        self.assertEqual(af.ton, 'unknown')
+        self.assertEqual(af.npi, 'unknown')
+        self.assertEqual(af.digits, '1234')
+
+    def test_fromBytes_odd(self):
+        af, trailer = AddressField.fromBytes('038021f399')
+        self.assertEqual(trailer, b'\x99')
+        self.assertEqual(af.ton, 'unknown')
+        self.assertEqual(af.npi, 'unknown')
+        self.assertEqual(af.digits, '123')
+
+    def test_toBytes(self):
+        encoded = h2b('04802143')
+        af, trailer = AddressField.fromBytes(encoded)
+        self.assertEqual(af.toBytes(), encoded)
+
+    def test_toBytes_odd(self):
+        af = AddressField('12345', 'international', 'isdn_e164')
+        encoded = af.toBytes()
+        self.assertEqual(encoded, h2b('05912143f5'))
+
+
+class Test_SUBMIT(unittest.TestCase):
+    def test_fromBytes(self):
+        s = SMS_SUBMIT.fromBytes('550d0b911614261771f000f5a78c0b050423f423f40003010201424547494e3a56434152440d0a56455253494f4e3a322e310d0a4e3a4d650d0a54454c3b505245463b43454c4c3b564f4943453a2b36313431363237313137300d0a54454c3b484f4d453b564f4943453a2b36313339353337303437310d0a54454c3b574f524b3b564f4943453a2b36313339363734373031350d0a454e443a')
+        self.assertEqual(s.tp_mti, 1)
+        self.assertEqual(s.tp_rd, True)
+        self.assertEqual(s.tp_vpf, 'relative')
+        self.assertEqual(s.tp_rp, False)
+        self.assertEqual(s.tp_udhi, True)
+        self.assertEqual(s.tp_srr, False)
+        self.assertEqual(s.tp_pid, 0)
+        self.assertEqual(s.tp_dcs, 0xf5)
+        self.assertEqual(s.tp_udl, 140)
+
+class Test_DELIVER(unittest.TestCase):
+    def test_fromBytes(self):
+        d = SMS_DELIVER.fromBytes('0408D0E5759A0E7FF6907090307513000824010101BB400101')
+        self.assertEqual(d.tp_mti, 0)
+        self.assertEqual(d.tp_mms, True)
+        self.assertEqual(d.tp_lp, False)
+        self.assertEqual(d.tp_rp, False)
+        self.assertEqual(d.tp_udhi, False)
+        self.assertEqual(d.tp_sri, False)
+        self.assertEqual(d.tp_pid, 0x7f)
+        self.assertEqual(d.tp_dcs, 0xf6)
+        self.assertEqual(d.tp_udl, 8)

tests/test_tlv.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from pySim.tlv import *
+
+class TestUtils(unittest.TestCase):
+    def test_camel_to_snake(self):
+        cases = [
+            ('CamelCase', 'camel_case'),
+            ('CamelCaseUPPER', 'camel_case_upper'),
+            ('Camel_CASE_underSCORE', 'camel_case_under_score'),
+        ]
+        for c in cases:
+            self.assertEqual(camel_to_snake(c[0]), c[1])
+
+    def test_flatten_dict_lists(self):
+        inp = [
+                { 'first': 1 },
+                { 'second': 2 },
+                { 'third': 3 },
+                ]
+        out = { 'first': 1, 'second':2, 'third': 3}
+        self.assertEqual(flatten_dict_lists(inp), out)
+
+    def test_flatten_dict_lists_nodict(self):
+        inp = [
+                { 'first': 1 },
+                { 'second': 2 },
+                { 'third': 3 },
+                4,
+                ]
+        self.assertEqual(flatten_dict_lists(inp), inp)
+
+    def test_flatten_dict_lists_nested(self):
+        inp = {'top': [
+                { 'first': 1 },
+                { 'second': 2 },
+                { 'third': 3 },
+                ] }
+        out = {'top': { 'first': 1, 'second':2, 'third': 3 } }
+        self.assertEqual(flatten_dict_lists(inp), out)
+
+class TestTranscodable(unittest.TestCase):
+    class XC_constr_class(Transcodable):
+        _construct = Int8ub
+        def __init__(self):
+            super().__init__();
+
+    def test_XC_constr_class(self):
+        """Transcodable derived class with _construct class variable"""
+        xc = TestTranscodable.XC_constr_class()
+        self.assertEqual(xc.from_bytes(b'\x23'), 35)
+        self.assertEqual(xc.to_bytes(), b'\x23')
+
+    class XC_constr_instance(Transcodable):
+        def __init__(self):
+            super().__init__();
+            self._construct = Int8ub
+
+    def test_XC_constr_instance(self):
+        """Transcodable derived class with _construct instance variable"""
+        xc = TestTranscodable.XC_constr_instance()
+        self.assertEqual(xc.from_bytes(b'\x23'), 35)
+        self.assertEqual(xc.to_bytes(), b'\x23')
+
+    class XC_method_instance(Transcodable):
+        def __init__(self):
+            super().__init__();
+        def _from_bytes(self, do):
+            return ('decoded', do)
+        def _to_bytes(self):
+            return self.decoded[1]
+
+    def test_XC_method_instance(self):
+        """Transcodable derived class with _{from,to}_bytes() methods"""
+        xc = TestTranscodable.XC_method_instance()
+        self.assertEqual(xc.to_bytes(), b'')
+        self.assertEqual(xc.from_bytes(b''), None)
+        self.assertEqual(xc.from_bytes(b'\x23'), ('decoded', b'\x23'))
+        self.assertEqual(xc.to_bytes(), b'\x23')
+
+class TestIE(unittest.TestCase):
+    class MyIE(IE, tag=0x23, desc='My IE description'):
+        _construct = Int8ub
+        def to_ie(self):
+            return self.to_bytes()
+
+    def test_IE_empty(self):
+        ie = TestIE.MyIE()
+        self.assertEqual(ie.to_dict(), {'my_ie': None})
+        self.assertEqual(repr(ie), 'MyIE(None)')
+        self.assertEqual(ie.is_constructed(), False)
+
+    def test_IE_from_bytes(self):
+        ie = TestIE.MyIE()
+        ie.from_bytes(b'\x42')
+        self.assertEqual(ie.to_dict(), {'my_ie': 66})
+        self.assertEqual(repr(ie), 'MyIE(66)')
+        self.assertEqual(ie.is_constructed(), False)
+        self.assertEqual(ie.to_bytes(), b'\x42')
+        self.assertEqual(ie.to_ie(), b'\x42')
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_utils.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim import utils
+from pySim.ts_31_102 import EF_SUCI_Calc_Info
+
+# we don't really want to thest TS 102 221, but the underlying DataObject codebase
+from pySim.ts_102_221 import AM_DO_EF, AM_DO_DF, SC_DO
+
+class DoTestCase(unittest.TestCase):
+
+    def testSeqOfChoices(self):
+        """A sequence of two choices with each a variety of DO/TLVs"""
+        arr_seq = utils.DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        # input data
+        dec_in = [{'access_mode': ['update_erase', 'read_search_compare']}, {'control_reference_template':'PIN1'}]
+        # encode it once
+        encoded = arr_seq.encode(dec_in)
+        # decode again
+        re_decoded = arr_seq.decode(encoded)
+        self.assertEqual(dec_in, re_decoded[0])
+
+class DecTestCase(unittest.TestCase):
+	# TS33.501 Annex C.4 test keys
+	hnet_pubkey_profile_b = "0272DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD1" # ID 27 in test file
+	hnet_pubkey_profile_a = "5A8D38864820197C3394B92613B20B91633CBD897119273BF8E4A6F4EEC0A650" # ID 30 in test file  
+
+	# TS31.121 4.9.4 EF_SUCI_Calc_Info test file
+	testfile_suci_calc_info = "A006020101020000A14B80011B8121" +hnet_pubkey_profile_b +"80011E8120" +hnet_pubkey_profile_a
+
+	decoded_testfile_suci = {
+		'prot_scheme_id_list': [
+			{'priority': 0, 'identifier': 2, 'key_index': 1},
+			{'priority': 1, 'identifier': 1, 'key_index': 2},
+			{'priority': 2, 'identifier': 0, 'key_index': 0}],
+		'hnet_pubkey_list': [
+			{'hnet_pubkey_identifier': 27, 'hnet_pubkey': hnet_pubkey_profile_b.lower()}, # because h2b/b2h returns all lower-case
+			{'hnet_pubkey_identifier': 30, 'hnet_pubkey': hnet_pubkey_profile_a.lower()}]
+	}
+
+	def testSplitHexStringToListOf5ByteEntries(self):
+		input_str = "ffffff0003ffffff0002ffffff0001"
+		expected = [
+			"ffffff0003",
+			"ffffff0002",
+			"ffffff0001",
+		]
+		self.assertEqual(utils.hexstr_to_Nbytearr(input_str, 5), expected)
+
+	def testDecMCCfromPLMN(self):
+		self.assertEqual(utils.dec_mcc_from_plmn("92f501"), 295)
+
+	def testDecMCCfromPLMN_unused(self):
+		self.assertEqual(utils.dec_mcc_from_plmn("ff0f00"), 4095)
+
+	def testDecMCCfromPLMN_str(self):
+		self.assertEqual(utils.dec_mcc_from_plmn_str("92f501"), "295")
+
+	def testDecMCCfromPLMN_unused_str(self):
+		self.assertEqual(utils.dec_mcc_from_plmn_str("ff0f00"), "")
+
+	def testDecMNCfromPLMN_twoDigitMNC(self):
+		self.assertEqual(utils.dec_mnc_from_plmn("92f501"), 10)
+
+	def testDecMNCfromPLMN_threeDigitMNC(self):
+		self.assertEqual(utils.dec_mnc_from_plmn("031263"), 361)
+
+	def testDecMNCfromPLMN_unused(self):
+		self.assertEqual(utils.dec_mnc_from_plmn("00f0ff"), 4095)
+
+	def testDecMNCfromPLMN_twoDigitMNC_str(self):
+		self.assertEqual(utils.dec_mnc_from_plmn_str("92f501"), "10")
+
+	def testDecMNCfromPLMN_threeDigitMNC_str(self):
+		self.assertEqual(utils.dec_mnc_from_plmn_str("031263"), "361")
+
+	def testDecMNCfromPLMN_unused_str(self):
+		self.assertEqual(utils.dec_mnc_from_plmn_str("00f0ff"), "")
+
+	def test_enc_plmn(self):
+		with self.subTest("2-digit MCC"):
+			self.assertEqual(utils.enc_plmn("001", "01F"), "00F110")
+			self.assertEqual(utils.enc_plmn("001", "01"), "00F110")
+			self.assertEqual(utils.enc_plmn("295", "10"), "92F501")
+
+		with self.subTest("3-digit MCC"):
+			self.assertEqual(utils.enc_plmn("001", "001"), "001100")
+			self.assertEqual(utils.enc_plmn("302", "361"), "031263")
+
+	def testDecAct_noneSet(self):
+		self.assertEqual(utils.dec_act("0000"), [])
+
+	def testDecAct_onlyUtran(self):
+		self.assertEqual(utils.dec_act("8000"), ["UTRAN"])
+
+	def testDecAct_onlyEUtran(self):
+		self.assertEqual(utils.dec_act("4000"), ["E-UTRAN"])
+
+	def testDecAct_onlyNgRan(self):
+		self.assertEqual(utils.dec_act("0800"), ["NG-RAN"])
+
+	def testDecAct_onlyGsm(self):
+		self.assertEqual(utils.dec_act("0080"), ["GSM"])
+
+	def testDecAct_onlyGsmCompact(self):
+		self.assertEqual(utils.dec_act("0040"), ["GSM COMPACT"])
+
+	def testDecAct_onlyCdma2000HRPD(self):
+		self.assertEqual(utils.dec_act("0020"), ["cdma2000 HRPD"])
+
+	def testDecAct_onlyCdma20001xRTT(self):
+		self.assertEqual(utils.dec_act("0010"), ["cdma2000 1xRTT"])
+
+	def testDecAct_allSet(self):
+		self.assertEqual(utils.dec_act("ffff"), ["UTRAN", "E-UTRAN WB-S1", "E-UTRAN NB-S1", "NG-RAN", "GSM", "GSM COMPACT", "cdma2000 HRPD", "cdma2000 1xRTT"])
+
+	def testDecxPlmn_w_act(self):
+		expected = {'mcc': '295', 'mnc': '10', 'act': ["UTRAN"]}
+		self.assertEqual(utils.dec_xplmn_w_act("92f5018000"), expected)
+
+	def testFormatxPlmn_w_act(self):
+		input_str = "92f501800092f5508000ffffff0000ffffff0000ffffff0000ffffff0000ffffff0000ffffff0000ffffff0000ffffff0000"
+		expected  = "\t92f5018000 # MCC: 295 MNC: 10 AcT: UTRAN\n"
+		expected += "\t92f5508000 # MCC: 295 MNC: 05 AcT: UTRAN\n"
+		expected += "\tffffff0000 # unused\n"
+		expected += "\tffffff0000 # unused\n"
+		expected += "\tffffff0000 # unused\n"
+		expected += "\tffffff0000 # unused\n"
+		expected += "\tffffff0000 # unused\n"
+		expected += "\tffffff0000 # unused\n"
+		expected += "\tffffff0000 # unused\n"
+		expected += "\tffffff0000 # unused\n"
+		self.assertEqual(utils.format_xplmn_w_act(input_str), expected)
+
+
+	def testDecodeSuciCalcInfo(self):
+		suci_calc_info = EF_SUCI_Calc_Info()
+		decoded = suci_calc_info._decode_hex(self.testfile_suci_calc_info)
+		self.assertDictEqual(self.decoded_testfile_suci, decoded)
+
+	def testEncodeSuciCalcInfo(self):
+		suci_calc_info = EF_SUCI_Calc_Info()
+		encoded = suci_calc_info._encode_hex(self.decoded_testfile_suci)
+		self.assertEqual(encoded.lower(), self.testfile_suci_calc_info.lower())
+
+	def testEnc_msisdn(self):
+		msisdn_encoded = utils.enc_msisdn("+4916012345678", npi=0x01, ton=0x03)
+		self.assertEqual(msisdn_encoded, "0891946110325476f8ffffffffff")
+		msisdn_encoded = utils.enc_msisdn("123456", npi=0x01, ton=0x03)
+		self.assertEqual(msisdn_encoded, "04b1214365ffffffffffffffffff")
+		msisdn_encoded = utils.enc_msisdn("12345678901234567890", npi=0x01, ton=0x03)
+		self.assertEqual(msisdn_encoded, "0bb121436587092143658709ffff")
+		msisdn_encoded = utils.enc_msisdn("+12345678901234567890", npi=0x01, ton=0x03)
+		self.assertEqual(msisdn_encoded, "0b9121436587092143658709ffff")
+		msisdn_encoded = utils.enc_msisdn("", npi=0x01, ton=0x03)
+		self.assertEqual(msisdn_encoded, "ffffffffffffffffffffffffffff")
+		msisdn_encoded = utils.enc_msisdn("+", npi=0x01, ton=0x03)
+		self.assertEqual(msisdn_encoded, "ffffffffffffffffffffffffffff")
+
+	def testDec_msisdn(self):
+		msisdn_decoded = utils.dec_msisdn("0891946110325476f8ffffffffff")
+		self.assertEqual(msisdn_decoded, (1, 1, "+4916012345678"))
+		msisdn_decoded = utils.dec_msisdn("04b1214365ffffffffffffffffff")
+		self.assertEqual(msisdn_decoded, (1, 3, "123456"))
+		msisdn_decoded = utils.dec_msisdn("0bb121436587092143658709ffff")
+		self.assertEqual(msisdn_decoded, (1, 3, "12345678901234567890"))
+		msisdn_decoded = utils.dec_msisdn("ffffffffffffffffffffffffffff")
+		self.assertEqual(msisdn_decoded, None)
+		msisdn_decoded = utils.dec_msisdn("00112233445566778899AABBCCDDEEFF001122330bb121436587092143658709ffff")
+		self.assertEqual(msisdn_decoded, (1, 3, "12345678901234567890"))
+		msisdn_decoded = utils.dec_msisdn("ffffffffffffffffffffffffffffffffffffffff0bb121436587092143658709ffff")
+		self.assertEqual(msisdn_decoded, (1, 3, "12345678901234567890"))
+
+class TestBerTlv(unittest.TestCase):
+    def test_BerTlvTagDec(self):
+        res = utils.bertlv_parse_tag(b'\x01')
+        self.assertEqual(res, ({'tag':1, 'constructed':False, 'class': 0}, b''))
+        res = utils.bertlv_parse_tag(b'\x21')
+        self.assertEqual(res, ({'tag':1, 'constructed':True, 'class': 0}, b''))
+        res = utils.bertlv_parse_tag(b'\x81\x23')
+        self.assertEqual(res, ({'tag':1, 'constructed':False, 'class': 2}, b'\x23'))
+        res = utils.bertlv_parse_tag(b'\x1f\x8f\x00\x23')
+        self.assertEqual(res, ({'tag':0xf<<7, 'constructed':False, 'class': 0}, b'\x23'))
+
+    def test_BerTlvLenDec(self):
+        self.assertEqual(utils.bertlv_encode_len(1), b'\x01')
+        self.assertEqual(utils.bertlv_encode_len(127), b'\x7f')
+        self.assertEqual(utils.bertlv_encode_len(128), b'\x81\x80')
+        self.assertEqual(utils.bertlv_encode_len(0x123456), b'\x83\x12\x34\x56')
+
+    def test_BerTlvLenEnc(self):
+        self.assertEqual(utils.bertlv_parse_len(b'\x01\x23'), (1, b'\x23'))
+        self.assertEqual(utils.bertlv_parse_len(b'\x7f'), (127, b''))
+        self.assertEqual(utils.bertlv_parse_len(b'\x81\x80'), (128, b''))
+        self.assertEqual(utils.bertlv_parse_len(b'\x83\x12\x34\x56\x78'), (0x123456, b'\x78'))
+
+    def test_BerTlvParseOne(self):
+        res = utils.bertlv_parse_one(b'\x81\x01\x01');
+        self.assertEqual(res, ({'tag':1, 'constructed':False, 'class':2}, 1, b'\x01', b''))
+
+class TestComprTlv(unittest.TestCase):
+    def test_ComprTlvTagDec(self):
+        res = utils.comprehensiontlv_parse_tag(b'\x12\x23')
+        self.assertEqual(res, ({'tag': 0x12, 'comprehension': False}, b'\x23'))
+        res = utils.comprehensiontlv_parse_tag(b'\x92')
+        self.assertEqual(res, ({'tag': 0x12, 'comprehension': True}, b''))
+        res = utils.comprehensiontlv_parse_tag(b'\x7f\x12\x34')
+        self.assertEqual(res, ({'tag': 0x1234, 'comprehension': False}, b''))
+        res = utils.comprehensiontlv_parse_tag(b'\x7f\x82\x34\x56')
+        self.assertEqual(res, ({'tag': 0x234, 'comprehension': True}, b'\x56'))
+
+    def test_ComprTlvTagEnc(self):
+        res  = utils.comprehensiontlv_encode_tag(0x12)
+        self.assertEqual(res, b'\x12')
+        res  = utils.comprehensiontlv_encode_tag({'tag': 0x12})
+        self.assertEqual(res, b'\x12')
+        res  = utils.comprehensiontlv_encode_tag({'tag': 0x12, 'comprehension':True})
+        self.assertEqual(res, b'\x92')
+        res  = utils.comprehensiontlv_encode_tag(0x1234)
+        self.assertEqual(res, b'\x7f\x12\x34')
+        res  = utils.comprehensiontlv_encode_tag({'tag': 0x1234, 'comprehension':True})
+        self.assertEqual(res, b'\x7f\x92\x34')
+
+if __name__ == "__main__":
+	unittest.main()

vpcd2smpp.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+#
+# This program receive APDUs via the VPCD protocol of Frank Morgner's
+# virtualsmartcard, encrypts them with OTA (over the air) keys and
+# forwards them via SMPP to a SMSC (SMS service centre).
+#
+# In other words, you can use it as a poor man's OTA server, to enable
+# you to use unmodified application software with PC/SC support to talk
+# securely via OTA with a remote SMS card.
+#
+# This is very much a work in progress at this point.
+
+#######################################################################
+# twisted VPCD Library
+#######################################################################
+
+import logging
+import struct
+import abc
+from typing import Union, Optional
+from construct import Struct, Int8ub, Int16ub, If, Enum, Bytes, this, len_, Rebuild
+from twisted.internet.protocol import Protocol, ReconnectingClientFactory
+from pySim.utils import b2h, h2b
+
+logger = logging.getLogger(__name__)
+
+class VirtualCard(abc.ABC):
+    """Abstract base class for a virtual smart card."""
+    def __init__(self, atr: Union[str, bytes]):
+        if isinstance(atr, str):
+            atr = h2b(atr)
+        self.atr = atr
+
+    @abc.abstractmethod
+    def power_change(self, new_state: bool):
+        """Power the card on or off."""
+        pass
+
+    @abc.abstractmethod
+    def reset(self):
+        """Reset the card."""
+        pass
+
+    @abc.abstractmethod
+    def rx_c_apdu(self, apdu: bytes):
+        """Receive a C-APDU from the reader/application."""
+        pass
+
+    def tx_r_apdu(self, apdu: Union[str, bytes]):
+        if isinstance(apdu, str):
+            apdu = h2b(apdu)
+        logger.info("R-APDU: %s" % b2h(apdu))
+        self.protocol.send_data(apdu)
+
+class VpcdProtocolBase(Protocol):
+    # Prefixed couldn't be used as the this.length wouldn't be available in this case
+    construct = Struct('length'/Rebuild(Int16ub, len_(this.data) + len_(this.ctrl)),
+                       'data'/If(this.length > 1, Bytes(this.length)),
+                       'ctrl'/If(this.length == 1, Enum(Int8ub, off=0, on=1, reset=2, atr=4)))
+    def __init__(self, vcard: VirtualCard):
+        self.recvBuffer = b''
+        self.connectionCorrupted = False
+        self.pduReadTimer = None
+        self.pduReadTimerSecs = 10
+        self.callLater = reactor.callLater
+        self.on = False
+        self.vcard = vcard
+        self.vcard.protocol = self
+
+    def dataReceived(self, data: bytes):
+        """entry point where twisted tells us data was received."""
+        #logger.debug('Data received: %s' % b2h(data))
+        self.recvBuffer = self.recvBuffer + data
+        while True:
+            if self.connectionCorrupted:
+                return
+            msg = self.readMessage()
+            if msg is None:
+                break
+            self.endPDURead()
+            self.rawMessageReceived(msg)
+
+        if len(self.recvBuffer) > 0:
+            self.incompletePDURead()
+
+    def incompletePDURead(self):
+        """We have an incomplete PDU in readBuffer, schedule pduReadTimer"""
+        if self.pduReadTimer and self.pduReadTimer.active():
+            return
+        self.pduReadTimer = self.callLater(self.pduReadTimerSecs, self.onPDUReadTimeout)
+
+    def endPDURead(self):
+        """We completed reading a PDU, cancel the pduReadTimer."""
+        if self.pduReadTimer and self.pduReadTimer.active():
+            self.pduReadTimer.cancel()
+
+    def readMessage(self) -> Optional[bytes]:
+        """read an entire [raw] message."""
+        pduLen = self._getMessageLength()
+        if pduLen is None:
+            return None
+        return self._getMessage(pduLen)
+
+    def _getMessageLength(self) -> Optional[int]:
+        if len(self.recvBuffer) < 2:
+            return None
+        return struct.unpack('!H', self.recvBuffer[:2])[0]
+
+    def _getMessage(self, pduLen: int) -> Optional[bytes]:
+        if len(self.recvBuffer) < pduLen+2:
+            return None
+
+        message = self.recvBuffer[:pduLen+2]
+        self.recvBuffer = self.recvBuffer[pduLen+2:]
+        return message
+
+    def onPDUReadTimeout(self):
+        logger.error('PDU read timed out. Buffer is now considered corrupt')
+        #self.coruptDataReceived
+
+    def rawMessageReceived(self, message: bytes):
+        """Called once a complete binary vpcd message has been received."""
+        pdu = None
+        try:
+            pdu = VpcdProtocolBase.construct.parse(message)
+        except Exception as e:
+            logger.exception(e)
+            logger.critical('Received corrupt PDU %s' % b2h(message))
+            #self.corupDataRecvd()
+        else:
+            self.PDUReceived(pdu)
+
+    def PDUReceived(self, pdu):
+        logger.debug("Rx PDU: %s" % pdu)
+        if pdu['data']:
+            return self.on_rx_data(pdu)
+        else:
+            method = getattr(self, 'on_rx_' + pdu['ctrl'])
+            return method(pdu)
+
+    def on_rx_atr(self, pdu):
+        self.send_data(self.vcard.atr)
+
+    def on_rx_on(self, pdu):
+        if self.on:
+            return
+        else:
+            self.on = True
+            self.vcard.power_change(self.on)
+
+    def on_rx_reset(self, pdu):
+        self.vcard.reset()
+
+    def on_rx_off(self, pdu):
+        if not self.on:
+            return
+        else:
+            self.on = False
+            self.vcard.power_change(self.on)
+
+    def on_rx_data(self, pdu):
+        self.vcard.rx_c_apdu(pdu['data'])
+
+    def send_pdu(self, pdu):
+        logger.debug("Sending PDU: %s" % pdu)
+        encoded = VpcdProtocolBase.construct.build(pdu)
+        #logger.debug("Sending binary: %s" % b2h(encoded))
+        self.transport.write(encoded)
+
+    def send_data(self, data: Union[str, bytes]):
+        if isinstance(data, str):
+            data = h2b(data)
+        return self.send_pdu({'length': 0, 'ctrl': '', 'data': data})
+
+    def send_ctrl(self, ctrl: str):
+        return self.send_pdu({'length': 0, 'ctrl': ctrl, 'data': ''})
+
+
+class VpcdProtocolClient(VpcdProtocolBase):
+    pass
+
+
+class VpcdClientFactory(ReconnectingClientFactory):
+    def __init__(self, vcard_class: VirtualCard):
+        self.vcard_class = vcard_class
+
+    def startedConnecting(self, connector):
+        logger.debug('Started to connect')
+
+    def buildProtocol(self, addr):
+        logger.info('Connection established to %s' % addr)
+        self.resetDelay()
+        return VpcdProtocolClient(vcard = self.vcard_class())
+
+    def clientConnectionLost(self, connector, reason):
+        logger.warning('Connection lost (reason: %s)' % reason)
+        super().clientConnectionLost(connector, reason)
+
+    def clientConnectionFailed(self, connector, reason):
+        logger.warning('Connection failed (reason: %s)' % reason)
+        super().clientConnectionFailed(connector, reason)
+
+#######################################################################
+# Application
+#######################################################################
+
+from pprint import pprint as pp
+
+from twisted.internet.protocol import Protocol, ReconnectingClientFactory, ClientCreator
+from twisted.internet import reactor
+
+from smpp.twisted.client import SMPPClientTransceiver, SMPPClientService
+from smpp.twisted.protocol import SMPPClientProtocol
+from smpp.twisted.config import SMPPClientConfig
+from smpp.pdu.operations import SubmitSM, DeliverSM
+from smpp.pdu import pdu_types
+
+from pySim.ota import OtaKeyset, OtaDialectSms
+from pySim.utils import b2h, h2b
+
+
+class MyVcard(VirtualCard):
+    def __init__(self, **kwargs):
+        super().__init__(atr='3B9F96801FC78031A073BE21136743200718000001A5', **kwargs)
+        self.smpp_client = None
+        # KIC1 + KID1 of 8988211000000467285
+        KIC1 = h2b('D0FDA31990D8D64178601317191669B4')
+        KID1 = h2b('D24EB461799C5E035C77451FD9404463')
+        KIC3 = h2b('C21DD66ACAC13CB3BC8B331B24AFB57B')
+        KID3 = h2b('12110C78E678C25408233076AA033615')
+        self.ota_keyset = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3, kic=KIC3,
+                                    algo_auth='triple_des_cbc2', kid_idx=3, kid=KID3)
+        self.ota_dialect = OtaDialectSms()
+        self.tar = h2b('B00011')
+        self.spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+                    'por_shall_be_ciphered':True, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+
+    def ensure_smpp(self):
+        config = SMPPClientConfig(host='localhost', port=2775, username='test', password='test')
+        if self.smpp_client:
+            return
+        self.smpp_client = SMPPClientTransceiver(config, self.handleSmpp)
+        smpp = self.smpp_client.connectAndBind()
+        #self.smpp = ClientCreator(reactor, SMPPClientProtocol, config, self.handleSmpp)
+        #d = self.smpp.connectTCP(config.host, config.port)
+        #d = self.smpp.connectAndBind()
+        #d.addCallback(self.forwardToClient, self.smpp)
+
+    def power_change(self, new_state: bool):
+        if new_state:
+            logger.info("POWER ON")
+            self.ensure_smpp()
+        else:
+            logger.info("POWER OFF")
+
+    def reset(self):
+        logger.info("RESET")
+
+    def rx_c_apdu(self, apdu: bytes):
+        pp(self.smpp_client.smpp)
+        logger.info("C-APDU: %s" % b2h(apdu))
+        # translate to Secured OTA RFM
+        secured = self.ota_dialect.encode_cmd(self.ota_keyset, self.tar, self.spi, apdu=apdu)
+        # add user data header
+        tpdu = b'\x02\x70\x00' + secured
+        # send via SMPP
+        self.tx_sms_tpdu(tpdu)
+        #self.tx_r_apdu('9000')
+
+    def tx_sms_tpdu(self, tpdu: bytes):
+        """Send a SMS TPDU via SMPP SubmitSM."""
+        dcs = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT,
+                                       gsmFeatures=[pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET])
+        submit = SubmitSM(source_addr='12',destination_addr='23', data_coding=dcs, esm_class=esm_class,
+                          protocol_id=0x7f, short_message=tpdu)
+        self.smpp_client.smpp.sendDataRequest(submit)
+
+    def handleSmpp(self, smpp, pdu):
+        #logger.info("Received SMPP %s" % pdu)
+        data = pdu.params['short_message']
+        #logger.info("Received SMS Data %s" % b2h(data))
+        r = self.ota_dialect.decode_resp(self.ota_keyset, self.spi, data)
+        logger.info("Decoded SMPP %s" % r)
+        self.tx_r_apdu(r['last_response_data'] + r['last_status_word'])
+
+
+if __name__ == '__main__':
+    import logging
+    logger = logging.getLogger(__name__)
+    import colorlog
+    log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+    colorlog.basicConfig(level=logging.INFO, format = log_format)
+    logger = colorlog.getLogger()
+
+    from twisted.internet import reactor
+    host = 'localhost'
+    port = 35963
+    reactor.connectTCP(host, port, VpcdClientFactory(vcard_class=MyVcard))
+    reactor.run()