RTC Decrypter

[Cycl0ne]

Is it allowed to share the decryption algorithm which RTC uses for the .dat files to be posted here? Since there are no updates, maybe someone is interessted? I stumbled upon it, cause i wanted some graphics out of it.

[ChristopheF]

The RTC project has been dead for so long, I believe you can safely post your findings.

[Cycl0ne]

Code: Select all

"""
Decrypt Return To Chaos (RTC) `Data/*.dat` files.

The algorithm was extracted from `RTCEditor.exe` (function at VA 0x4d7150,
called from the per-file loaders at 0x4bb5c0/0x4bb610/... e.g. for
`Data\\Graphics1.dat`).

File layout (little-endian):
    offset 0  : uint32  checksum     (verified after decryption)
    offset 4  : uint32  key          (initial cipher state)
    offset 8  : uint8[] payload      (XOR stream cipher, see below)

Stream cipher (per byte i of the payload, key updated each step):

    s = i & 3
    switch (s):
        0:  byte = key & 0xff;              key = key + 1
        1:  byte = (key >> 8) & 0xff;       key = key * 13
        2:  byte = (key >> 16) & 0xff;      key = key ^ (byte * 0x01010101)
        3:  byte = (key >> 24) & 0xff;      key = key - 1
    payload[i] ^= byte

All arithmetic is mod 2^32.  `byte` is taken from the *old* key, then the
key is updated.

Checksum (over the decrypted payload):

    c = 0x47102
    for i, b in enumerate(payload):
        c += sign_extend_byte(b) << (i & 31)   # mod 2^32
    assert c == stored_checksum

A decrypted payload begins with its own filename (NUL terminated) followed by
a uint16 count, then tagged records (e.g. "BMC"/"BMV"/"BM1"/"BM3"/"BM6" for
the Graphics#.dat files).
"""

from __future__ import annotations

import argparse
import struct
import sys
from pathlib import Path

MASK32 = 0xFFFFFFFF
CHECKSUM_SEED = 0x47102


def decrypt(payload: bytes, key: int) -> bytearray:
    """Apply RTC's XOR stream cipher to `payload` using `key`."""
    out = bytearray(payload)
    k = key & MASK32
    for i in range(len(out)):
        s = i & 3
        if s == 0:
            b = k & 0xFF
            k = (k + 1) & MASK32
        elif s == 1:
            b = (k >> 8) & 0xFF
            k = (k * 13) & MASK32
        elif s == 2:
            b = (k >> 16) & 0xFF
            k = (k ^ (b * 0x01010101)) & MASK32
        else:  # s == 3
            b = (k >> 24) & 0xFF
            k = (k - 1) & MASK32
        out[i] ^= b
    return out


def compute_checksum(data: bytes) -> int:
    """RTC payload checksum: 0x47102 + sum( signed_byte << (i & 31) )."""
    c = CHECKSUM_SEED
    for i, b in enumerate(data):
        signed = b - 0x100 if b & 0x80 else b
        c = (c + (signed << (i & 31))) & MASK32
    return c


def parse_header(raw: bytes) -> tuple[int, int]:
    if len(raw) < 8:
        raise ValueError("file too small to contain an RTC header")
    checksum, key = struct.unpack("<II", raw[:8])
    return checksum, key


def read_filename(decrypted: bytes) -> str:
    """Decrypted payloads start with their own NUL-terminated filename."""
    nul = decrypted.find(b"\x00")
    return decrypted[:nul].decode("latin1") if nul >= 0 else ""


def decrypt_file(path: Path, verify: bool = True) -> tuple[bytearray, bool]:
    raw = path.read_bytes()
    checksum, key = parse_header(raw)
    payload = decrypt(raw[8:], key)
    ok = (compute_checksum(payload) == checksum)
    if verify and not ok:
        raise ValueError(
            f"{path}: checksum mismatch (stored={checksum:#010x}, "
            f"computed={compute_checksum(payload):#010x})"
        )
    return payload, ok


def main(argv: list[str] | None = None) -> int:
    p = argparse.ArgumentParser(
        description="Decrypt Return To Chaos Data/*.dat files"
    )
    p.add_argument(
        "files",
        nargs="+",
        type=Path,
        help="one or more *.dat files (e.g. Data/Graphics1.dat)",
    )
    p.add_argument(
        "-o",
        "--outdir",
        type=Path,
        default=None,
        help="write decrypted payload as <name>.dec (default: beside input)",
    )
    p.add_argument(
        "--no-verify",
        action="store_true",
        help="skip checksum verification",
    )
    p.add_argument(
        "-q",
        "--quiet",
        action="store_true",
        help="only print errors",
    )
    args = p.parse_args(argv)

    rc = 0
    for f in args.files:
        if not f.is_file():
            print(f"error: {f}: not a file", file=sys.stderr)
            rc = 1
            continue
        try:
            payload, ok = decrypt_file(f, verify=not args.no_verify)
        except ValueError as e:
            print(f"error: {e}", file=sys.stderr)
            rc = 1
            continue

        if not args.quiet:
            name = read_filename(payload)
            print(
                f"{f}: ok={ok} bytes={len(payload)} "
                f"filename={name!r} head={bytes(payload[:16])!r}"
            )

        if args.outdir is not None:
            args.outdir.mkdir(parents=True, exist_ok=True)
            outpath = args.outdir / (f.name + ".dec")
        else:
            outpath = f.with_suffix(f.suffix + ".dec")
        outpath.write_bytes(payload)
        if not args.quiet:
            print(f"  -> wrote {outpath}")

    return rc


if __name__ == "__main__":
    raise SystemExit(main())

Have fun