Mach-O: Read the LC_UNIXTHREAD entry point per cputype

cle/backends/macho/macho.py

@@ -22,8 +22,8 @@
 from cle.errors import CLECompatibilityError, CLEInvalidBinaryError, CLEOperationError
 
 from .encrypted_sentinel_backer import CryptSentinel
+from .macho_enums import ARMThreadFlavor, CPUType, MachoFiletype, MH_flags, X86ThreadFlavor
 from .macho_enums import LoadCommands as LC
-from .macho_enums import MachoFiletype, MH_flags
 from .section import TYPE_MASK, ZEROFILL_SECTION_TYPES, MachOSection
 from .segment import MachOSegment
 from .structs import (
@@ -43,6 +43,16 @@
 
 __all__ = ("MachO", "MachOSection", "MachOSegment", "SymbolList")
 
+# Layout of the thread states LC_UNIXTHREAD can carry, keyed by (cputype, flavor) because a flavor number
+# only identifies a thread state together with the cputype it belongs to. Every format string stops at the
+# program counter, so the last field it unpacks is the entry point.
+UNIXTHREAD_PC_FORMATS = {
+    (CPUType.CPU_TYPE_X86, X86ThreadFlavor.x86_THREAD_STATE32): "11I",  # __eip
+    (CPUType.CPU_TYPE_X86_64, X86ThreadFlavor.x86_THREAD_STATE64): "17Q",  # __rip
+    (CPUType.CPU_TYPE_ARM, ARMThreadFlavor.ARM_THREAD_STATE): "16I",  # __pc
+    (CPUType.CPU_TYPE_ARM64, ARMThreadFlavor.ARM_THREAD_STATE64): "33Q",  # __pc
+}
+
 
 # pylint: disable=abstract-method
 class SymbolList(SortedKeyList):
@@ -179,15 +189,25 @@ def __init__(self, *args, **kwargs):
             # Note that this should be customized for Apple ABI (TODO)
             self.set_arch(archinfo.arch_from_id(arch_ident, endness="lsb" if self.struct_byteorder == "<" else "msb"))
 
+            # Start reading load commands
+            lc_offset = (7 if self.arch.bits == 32 else 8) * 4
+
             # Determine the base address the binary was linked against
             # and set the values for the Backend and Loader accordingly
             if self.pic and self.filetype == MachoFiletype.MH_EXECUTE:
                 assert self.is_main_bin, "An file of type MH_EXECUTE should be the main bin, this should not happen"
                 # a Position Independent Main binary would later be loaded at 0x400000, which isn't legal for Mach-O
-                # Also, its segment vaddrs are relative to 0x100000000, so we set this as the linked base
-                # and the MachO Backend code uses the AdressTranslator to translate linked addresses to relative ones
+                # An executable is linked at the address of its __TEXT segment: that is where the mach header
+                # itself ends up, and what __mh_execute_header resolves to. ld64 puts it at 0x100000000 on 64 bit
+                # and 0x4000 on 32 bit, but that is a default and not a property of the format -- Go's linker
+                # links darwin/amd64 executables at 0x1000000 -- so read it instead of assuming it, and fall back
+                # to the ld64 default for a binary that declares no __TEXT.
+                # The MachO Backend code uses the AddressTranslator to translate linked addresses to relative ones.
                 # In theory this is the place where the slide for rebasing should be added, but this isn't supported yet
-                if self.arch.bits == 64:
+                text_vmaddr = self._text_segment_vmaddr(lc_offset)
+                if text_vmaddr:
+                    self.linked_base = self.mapped_base = text_vmaddr
+                elif self.arch.bits == 64:
                     self.linked_base = self.mapped_base = 2**32
                 elif self.arch.bits == 32:
                     self.linked_base = self.mapped_base = 0x4000
@@ -223,9 +243,6 @@ def __init__(self, *args, **kwargs):
                     "Please open an issue if you need support for this"
                 )
 
-            # Start reading load commands
-            lc_offset = (7 if self.arch.bits == 32 else 8) * 4
-
             self._parse_load_commands(lc_offset)
 
         except OSError as e:
@@ -297,6 +314,30 @@ def check_compatibility(cls, spec, obj):
         # TODO: Check properly, but for now libs are just used via force load libs anyway
         return True
 
+    def _text_segment_vmaddr(self, lc_offset: int) -> int | None:
+        """
+        The address __TEXT is linked at, read straight from the load commands before they are parsed
+        properly, because the base address the rest of the parse works against depends on it.
+
+        :return: the vmaddr of the __TEXT segment, or None if the binary declares no __TEXT segment.
+        """
+        binary_file = self._binary_stream
+        count = 0
+        offset = lc_offset
+        # Bounded the same way _parse_load_commands bounds itself, so a header that lies about either
+        # ncmds or sizeofcmds cannot walk this loop off the end of the commands
+        while count < self.ncmds and (offset - lc_offset) < self.sizeofcmds:
+            count += 1
+            cmd, size = self._unpack("2I", binary_file, offset, 8)
+            if cmd in (LC.LC_SEGMENT, LC.LC_SEGMENT_64):
+                segname = self._read(binary_file, offset + 8, 16).rstrip(b"\0")
+                if segname == b"__TEXT":
+                    if cmd == LC.LC_SEGMENT_64:
+                        return self._unpack("Q", binary_file, offset + 24, 8)[0]
+                    return self._unpack("I", binary_file, offset + 24, 4)[0]
+            offset += size
+        return None
+
     def _parse_load_commands(self, lc_offset):
         # Possible optimization: Remove all unnecessary calls to seek()
         # Load commands have a common structure: First 4 bytes identify the command by a magic number
@@ -750,10 +791,10 @@ def _detect_arch_ident(self):
         try:
             arch_lookup = {
                 # contains all supported architectures. Note that apple deviates from standard ABI, see Apple docs
-                0x100000C: "aarch64",
-                0xC: "arm",
-                0x7: "x86",
-                0x1000007: "x64",
+                CPUType.CPU_TYPE_ARM64: "aarch64",
+                CPUType.CPU_TYPE_ARM: "arm",
+                CPUType.CPU_TYPE_X86: "x86",
+                CPUType.CPU_TYPE_X86_64: "x64",
             }
             return arch_lookup[self.cputype]  # subtype currently not needed
         except KeyError:
@@ -824,21 +865,32 @@ def _load_lc_unixthread(self, f, offset):
             )
 
         # parse basic structure
-        # _, cmdsize, flavor, long_count
-        _, _, flavor, _ = self._unpack("4I", f, offset, 16)
-
-        # we only support 4 different types of thread state atm
-        # TODO: This is the place to add x86 and x86_64 thread states
-        if flavor == 1 and self.arch.bits != 64:  # ARM_THREAD_STATE or ARM_UNIFIED_THREAD_STATE or ARM_THREAD_STATE32
-            blob = self._unpack("16I", f, offset + 16, 64)  # parses only until __pc
-        elif flavor == 1 and self.arch.bits == 64 or flavor == 6:
-            # ARM_THREAD_STATE or ARM_UNIFIED_THREAD_STATE or ARM_THREAD_STATE64
-            blob = self._unpack("33Q", f, offset + 16, 264)  # parses only until __pc
-        else:
-            log.error("Unknown thread flavor: %d", flavor)
-            raise CLECompatibilityError()
+        # _, cmdsize, flavor, count
+        _, _, flavor, count = self._unpack("4I", f, offset, 16)
+
+        # Nothing below can abort the load: the entry point is all LC_UNIXTHREAD contributes, and
+        # _resolve_entry already reports a binary that has no entry point at all.
+        fmt = UNIXTHREAD_PC_FORMATS.get((self.cputype, flavor))
+        if fmt is None:
+            # Without a known layout there is no telling where the program counter sits in the thread state.
+            log.warning("Unsupported thread state flavor %d for cputype %#x", flavor, self.cputype)
+            return
+
+        # "=" asks for the standard sizes, which is what both byteorder prefixes select, so the size
+        # of the thread state does not depend on which one this binary needs.
+        size = struct.calcsize("=" + fmt)
+        if count * 4 < size:
+            # count is the length of the thread state in 32 bit words. Unpacking more than the command
+            # declares would read whatever follows it rather than the program counter.
+            log.warning("LC_UNIXTHREAD declares %d words of thread state, too few for flavor %d", count, flavor)
+            return
+
+        state = self._read(f, offset + 16, size)
+        if len(state) < size:
+            log.warning("File ends inside the LC_UNIXTHREAD thread state")
+            return
 
-        self.unixthread_pc = blob[-1]
+        self.unixthread_pc = self._unpack_with_byteorder(fmt, state)[-1]
         log.debug("LC_UNIXTHREAD: __pc=%#x", self.unixthread_pc)
 
     def _load_dylib_info(self, f, offset):

cle/backends/macho/macho_enums.py

@@ -229,6 +229,43 @@ class MH_flags(IntEnum):
     MH_DYLIB_IN_CACHE = 0x80000000
 
 
+class CPUType(IntEnum):
+    """
+    from mach/machine.h
+
+    Values for the cputype field of the mach_header, limited to the architectures cle supports
+    """
+
+    CPU_TYPE_X86 = 0x7
+    CPU_TYPE_X86_64 = 0x1000007
+    CPU_TYPE_ARM = 0xC
+    CPU_TYPE_ARM64 = 0x100000C
+
+
+class X86ThreadFlavor(IntEnum):
+    """
+    from mach/i386/thread_status.h
+
+    Flavors of the thread state carried by LC_UNIXTHREAD on CPU_TYPE_X86 and CPU_TYPE_X86_64.
+    A flavor number only identifies a thread state together with the cputype it belongs to.
+    """
+
+    x86_THREAD_STATE32 = 1
+    x86_THREAD_STATE64 = 4
+
+
+class ARMThreadFlavor(IntEnum):
+    """
+    from mach/arm/thread_status.h
+
+    Flavors of the thread state carried by LC_UNIXTHREAD on CPU_TYPE_ARM and CPU_TYPE_ARM64.
+    A flavor number only identifies a thread state together with the cputype it belongs to.
+    """
+
+    ARM_THREAD_STATE = 1
+    ARM_THREAD_STATE64 = 6
+
+
 class RebaseType(IntEnum):
     """
     from mach-o/loader.h

tests/test_macho_unixthread.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+from __future__ import annotations
+
+import logging
+import struct
+import tempfile
+from pathlib import Path
+
+import cle
+from cle.backends.macho.macho_enums import LoadCommands as LC
+
+TEST_BASE = Path(__file__).resolve().parent.parent.parent / "binaries"
+
+# `terramate` out of the official Homebrew bottle for tenv 4.15.1, x86_64 macOS. Go's internal linker is
+# the toolchain still shipping LC_UNIXTHREAD instead of LC_MAIN, and it stores an x86_THREAD_STATE64
+# whose __rip is the entry point. `otool -l` puts the command at file offset 0x650, command 6 of 14.
+FIXTURE = TEST_BASE / "tests" / "x86_64" / "terramate.macho"
+UNIXTHREAD_OFFSET = 0x650
+FLAVOR_FIELD = 8
+COUNT_FIELD = 12
+ENTRY = 0x1081180
+SEGMENTS = ["__PAGEZERO", "__TEXT", "__DATA_CONST", "__DATA", "__LINKEDIT"]
+
+# x86_FLOAT_STATE64 from mach/i386/thread_status.h. A thread state LC_UNIXTHREAD is allowed to carry,
+# but one with no program counter in it, so there is nothing for the loader to take an entry point from.
+X86_FLOAT_STATE64 = 5
+
+
+def load(path: Path | str) -> cle.MachO:
+    ld = cle.Loader(str(path), main_opts={"backend": "mach-o"}, auto_load_libs=False)
+    assert isinstance(ld.main_object, cle.MachO)
+    return ld.main_object
+
+
+def patch_unixthread(field: int, value: int) -> Path:
+    """
+    A copy of the fixture with one 32 bit field of its LC_UNIXTHREAD command overwritten.
+
+    :param field:   Offset of the field within the command.
+    :param value:   Value to write there.
+    """
+    data = bytearray(FIXTURE.read_bytes())
+    command, _cmdsize = struct.unpack_from("<2I", data, UNIXTHREAD_OFFSET)
+    assert command == LC.LC_UNIXTHREAD, "the fixture's thread command moved, update UNIXTHREAD_OFFSET"
+    struct.pack_into("<I", data, UNIXTHREAD_OFFSET + field, value)
+    handle = tempfile.NamedTemporaryFile(suffix=".macho", delete=False)
+    with handle:
+        handle.write(data)
+    return Path(handle.name)
+
+
+def test_entry_point_comes_from_the_thread_state():
+    # Read as an ARM thread state this comes back with the last of the 16 words the command declares
+    # rather than __rip, and dispatching on the flavor alone used to refuse the binary outright.
+    obj = load(FIXTURE)
+    assert obj.arch.name == "AMD64"
+    assert obj.unixthread_pc == ENTRY
+    assert obj.entry == ENTRY
+
+
+def test_flavor_without_a_known_layout_still_loads():
+    path = patch_unixthread(FLAVOR_FIELD, X86_FLOAT_STATE64)
+    try:
+        obj = load(path)
+    finally:
+        path.unlink()
+    # The entry point is all the command contributes, so the rest of the binary still loads without it
+    assert obj.unixthread_pc is None
+    assert obj.entry == 0
+    assert [segment.segname for segment in obj.segments] == SEGMENTS
+
+
+def test_thread_state_shorter_than_its_flavor_still_loads():
+    # Two words is nowhere near an x86_thread_state64_t, so reading one would run past the command
+    path = patch_unixthread(COUNT_FIELD, 2)
+    try:
+        obj = load(path)
+    finally:
+        path.unlink()
+    assert obj.unixthread_pc is None
+    assert obj.entry == 0
+    assert [segment.segname for segment in obj.segments] == SEGMENTS
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    test_entry_point_comes_from_the_thread_state()
+    test_flavor_without_a_known_layout_still_loads()
+    test_thread_state_shorter_than_its_flavor_still_loads()