relayers new code

app/core/crypto/aes_gcm_siv_stream.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import hmac
+import hashlib
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from gcm_siv import GcmSiv  # requires `gcm_siv` package
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+SCHEME_AES_GCM_SIV = 0x01
+
+
+def _derive_nonce(salt: bytes, idx: int) -> bytes:
+    b = idx.to_bytes(8, 'big')
+    return hmac.new(salt, b, hashlib.sha256).digest()[:12]
+
+
+def build_header(chunk_bytes: int, salt: bytes) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # MAGIC(4) | ver(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt | reserved(5)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([SCHEME_AES_GCM_SIV])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream using AES-GCM-SIV per chunk with deterministic nonces.
+    Frame: [p_len:4][cipher][tag(16)].
+    """
+    yield build_header(chunk_bytes, salt)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        ct_and_tag = GcmSiv(key).encrypt(nonce, block, associated_data=None)
+        # Split tag
+        tag = ct_and_tag[-16:]
+        ct = ct_and_tag[:-16]
+        yield struct.pack(">I", len(block))
+        yield ct
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """Parse ENCF v1 (AES-GCM-SIV) and write plaintext to out_path."""
+    import aiofiles
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # header minimal
+    await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    if version != 1 or scheme != SCHEME_AES_GCM_SIV:
+        raise ValueError("unsupported encf header")
+    chunk_bytes = struct.unpack(">I", bytes(buf[6:10]))[0]
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    salt = bytes(buf[11:11 + salt_len])
+    del buf[:hdr_len]
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            await _fill(4)
+            if len(buf) == 0:
+                break
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            ct = bytes(buf[:p_len])
+            tag = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            pt = GcmSiv(key).decrypt(nonce, ct + tag, associated_data=None)
+            await out.write(pt)
+            idx += 1
+

app/core/crypto/aes_siv_stream.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import os
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from Crypto.Cipher import SIV
+from Crypto.Cipher import AES
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+
+# Scheme codes
+SCHEME_AES_SIV = 0x02  # RFC5297 AES-SIV (CMAC-based)
+
+
+def build_header(chunk_bytes: int, salt: bytes, scheme: int = SCHEME_AES_SIV) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # Layout: MAGIC(4) | version(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt(N) | reserved(5 zeros)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([scheme])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def parse_header(buf: bytes) -> tuple[int, int, int, bytes, int]:
+    if len(buf) < 4 + 1 + 1 + 4 + 1:
+        raise ValueError("header too short")
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    chunk_bytes = struct.unpack(">I", buf[6:10])[0]
+    salt_len = buf[10]
+    needed = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    if len(buf) < needed:
+        raise ValueError("incomplete header")
+    salt = buf[11:11 + salt_len]
+    # reserved 5 bytes at the end ignored
+    return version, scheme, chunk_bytes, salt, needed
+
+
+def _ad(salt: bytes, idx: int) -> bytes:
+    return salt + struct.pack(">Q", idx)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream bytes: [header] then for each chunk: [p_len:4][cipher][tag(16)].
+    Uses AES-SIV (RFC5297) with per-chunk associated data salt||index.
+    """
+    yield build_header(chunk_bytes, salt, SCHEME_AES_SIV)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        siv = SIV.new(key=key, ciphermod=AES)  # new object per message
+        siv.update(_ad(salt, idx))
+        ciph, tag = siv.encrypt_and_digest(block)
+        yield struct.pack(">I", len(block))
+        yield ciph
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Parse ENCF v1 stream from async byte iterator and write plaintext to out_path.
+    """
+    import aiofiles
+    from Crypto.Cipher import SIV as _SIV
+    from Crypto.Cipher import AES as _AES
+
+    buf = bytearray()
+
+    async def _fill(n: int):
+        """Ensure at least n bytes in buffer (or EOF)."""
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # Read and parse header
+    await _fill(4 + 1 + 1 + 4 + 1)  # minimal header
+    # Might still be incomplete if salt_len > 0; keep filling progressively
+    # First, get preliminary to know salt_len
+    if len(buf) < 11:
+        await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    version, scheme, chunk_bytes, salt, consumed = parse_header(bytes(buf))
+    del buf[:consumed]
+    if version != 1:
+        raise ValueError("unsupported ENCF version")
+    if scheme != SCHEME_AES_SIV:
+        raise ValueError("unsupported scheme")
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            # Need at least 4 bytes for p_len
+            await _fill(4)
+            if len(buf) == 0:
+                break  # EOF exactly on boundary
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            # Now need p_len + 16 bytes
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            c = bytes(buf[:p_len])
+            t = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            siv = _SIV.new(key=key, ciphermod=_AES)
+            siv.update(_ad(salt, idx))
+            p = siv.decrypt_and_verify(c, t)
+            await out.write(p)
+            idx += 1
+

app/core/crypto/aesgcm_stream.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import os
+import hmac
+import hashlib
+from typing import BinaryIO, Iterator
+
+from Crypto.Cipher import AES
+
+
+CHUNK_BYTES = int(os.getenv("CRYPTO_CHUNK_BYTES", "1048576"))  # 1 MiB
+
+
+def _derive_nonce(salt: bytes, chunk_index: int) -> bytes:
+    """Derive a 12-byte GCM nonce deterministically from per-file salt and chunk index."""
+    idx = chunk_index.to_bytes(8, 'big')
+    digest = hmac.new(salt, idx, hashlib.sha256).digest()
+    return digest[:12]
+
+
+def encrypt_stream_aesgcm(src: BinaryIO, key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Read plaintext from src by CHUNK_BYTES, encrypt each chunk with AES-GCM using a
+    deterministic nonce derived from (salt, index). Yields bytes in framing: [C_i][TAG_i]...
+    Ciphertext length equals plaintext chunk length. Tag is 16 bytes.
+    """
+    assert len(key) in (16, 24, 32)
+    assert len(salt) >= 12
+    idx = 0
+    while True:
+        block = src.read(CHUNK_BYTES)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+        ciphertext, tag = cipher.encrypt_and_digest(block)
+        yield ciphertext
+        yield tag
+        idx += 1
+
+
+def decrypt_stream_aesgcm_iter(byte_iter: Iterator[bytes], key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Decrypt a stream that was produced by encrypt_stream_aesgcm.
+    Frame format: concatenation of [C_i][TAG_i] for each i, where |C_i| = CHUNK_BYTES and |TAG_i|=16.
+    We accept arbitrary chunking from the underlying iterator and reframe accordingly.
+    """
+    assert len(key) in (16, 24, 32)
+    buf = bytearray()
+    idx = 0
+    TAG_LEN = 16
+    def _try_yield():
+        nonlocal idx
+        out = []
+        while len(buf) >= CHUNK_BYTES + TAG_LEN:
+            c = bytes(buf[:CHUNK_BYTES])
+            t = bytes(buf[CHUNK_BYTES:CHUNK_BYTES+TAG_LEN])
+            del buf[:CHUNK_BYTES+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+            try:
+                p = cipher.decrypt_and_verify(c, t)
+            except Exception as e:
+                raise ValueError(f"Decrypt failed at chunk {idx}: {e}")
+            out.append(p)
+            idx += 1
+        return out
+    for chunk in byte_iter:
+        if not chunk:
+            continue
+        buf.extend(chunk)
+        for p in _try_yield():
+            yield p
+    # At end, buffer must be empty
+    if len(buf) != 0:
+        raise ValueError("Trailing bytes in encrypted stream (incomplete frame)")
+

app/core/crypto/encf_stream.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import AsyncIterator
+
+from .aes_gcm_siv_stream import MAGIC as _MAGIC, VERSION as _VER, SCHEME_AES_GCM_SIV
+from .aes_gcm_siv_stream import decrypt_encf_to_file as _dec_gcmsiv
+from .aes_siv_stream import decrypt_encf_to_file as _dec_siv
+
+
+async def decrypt_encf_auto(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Detect scheme by peeking header, then delegate to proper decrypter.
+    Re-feeds the peeked bytes back to the chosen decoder.
+    """
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                ch = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if ch:
+                buf.extend(ch)
+
+    await _fill(11)
+    if buf[:4] != _MAGIC:
+        raise ValueError("bad magic")
+    scheme = buf[5]
+
+    async def _prepend_iter():
+        nonlocal buf
+        if buf:
+            yield bytes(buf)
+        async for ch in byte_iter:
+            yield ch
+
+    if scheme == SCHEME_AES_GCM_SIV:
+        await _dec_gcmsiv(_prepend_iter(), key, out_path)
+    else:
+        await _dec_siv(_prepend_iter(), key, out_path)
+

app/core/crypto/x25519.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import base64
+from typing import Tuple
+
+from nacl import public, signing, bindings
+
+
+def ed25519_to_x25519(ed_seed: bytes) -> Tuple[public.PrivateKey, public.PublicKey]:
+    """Convert Ed25519 seed (32 bytes) to X25519 key pair using libsodium conversion."""
+    if len(ed_seed) != 32:
+        raise ValueError("ed25519 seed must be 32 bytes")
+    sk_ed = signing.SigningKey(ed_seed)
+    sk_ed_bytes = sk_ed._seed + sk_ed.verify_key._key  # 64-byte expanded sk (seed||pub)
+    sk_x_bytes = bindings.crypto_sign_ed25519_sk_to_curve25519(sk_ed_bytes)
+    pk_x_bytes = bindings.crypto_sign_ed25519_pk_to_curve25519(bytes(sk_ed.verify_key))
+    sk_x = public.PrivateKey(sk_x_bytes)
+    pk_x = public.PublicKey(pk_x_bytes)
+    return sk_x, pk_x
+
+
+def x25519_pub_b64_from_ed_seed(ed_seed: bytes) -> str:
+    _, pk = ed25519_to_x25519(ed_seed)
+    return base64.b64encode(bytes(pk)).decode()
+