nice version
This commit is contained in:
parent
360f8110a4
commit
38e54f0ab2
|
|
@ -28,7 +28,7 @@ async def s_api_v1_content_list(request):
|
||||||
select(StoredContent)
|
select(StoredContent)
|
||||||
.where(
|
.where(
|
||||||
StoredContent.type.like(store + '%'),
|
StoredContent.type.like(store + '%'),
|
||||||
StoredContent.disabled == False
|
StoredContent.disabled.is_(None)
|
||||||
)
|
)
|
||||||
.order_by(StoredContent.created.desc())
|
.order_by(StoredContent.created.desc())
|
||||||
.offset(offset)
|
.offset(offset)
|
||||||
|
|
|
||||||
|
|
@ -27,14 +27,29 @@ async def s_api_v1_upload_tus_hook(request):
|
||||||
tusd HTTP hook endpoint. We mainly handle post-finish to: encrypt -> IPFS add+pin -> record DB.
|
tusd HTTP hook endpoint. We mainly handle post-finish to: encrypt -> IPFS add+pin -> record DB.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
payload: Dict[str, Any] = request.json or {}
|
payload: Dict[str, Any] = request.json
|
||||||
except Exception:
|
except Exception:
|
||||||
payload = {}
|
payload = None
|
||||||
event = payload.get("Type") or payload.get("type") or payload.get("Event") or payload.get("event")
|
if payload is None:
|
||||||
|
raw_body = request.body or b''
|
||||||
|
try:
|
||||||
|
payload = json.loads(raw_body) if raw_body else {}
|
||||||
|
except Exception:
|
||||||
|
payload = {}
|
||||||
|
event = (payload.get("Type") or payload.get("type") or
|
||||||
|
payload.get("Event") or payload.get("event") or
|
||||||
|
payload.get("Hook") or payload.get("hook") or
|
||||||
|
payload.get("HookName") or payload.get("hook_name") or
|
||||||
|
request.headers.get("Hook-Name") or request.headers.get("hook-name"))
|
||||||
upload = payload.get("Upload") or payload.get("upload") or {}
|
upload = payload.get("Upload") or payload.get("upload") or {}
|
||||||
|
|
||||||
if not event:
|
if not event:
|
||||||
return response.json({"ok": False, "error": "NO_EVENT"}, status=400)
|
hook_name = (payload.get("HookName") or payload.get("hook") or
|
||||||
|
payload.get("hook_name") or request.headers.get("Hook-Name"))
|
||||||
|
raw = request.body or b''
|
||||||
|
preview = raw[:512]
|
||||||
|
make_log("tus-hook", f"Missing event type in hook payload; ignoring (hook={hook_name}, keys={list(payload.keys())}, raw={preview!r})", level="warning")
|
||||||
|
return response.json({"ok": True, "skipped": True})
|
||||||
|
|
||||||
if event not in ("post-finish", "postfinish"):
|
if event not in ("post-finish", "postfinish"):
|
||||||
# accept but ignore other events
|
# accept but ignore other events
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import os
|
||||||
import json
|
import json
|
||||||
import shutil
|
import shutil
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import List, Tuple
|
from typing import List, Tuple, Optional
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
|
@ -57,7 +57,7 @@ async def _save_derivative(file_path: str, filename: str) -> Tuple[str, int]:
|
||||||
return file_hash, size
|
return file_hash, size
|
||||||
|
|
||||||
|
|
||||||
async def _run_media_converter(input_host_path: str, input_ext: str, quality: str, trim_value: str | None, is_audio: bool) -> Tuple[str, dict]:
|
async def _run_media_converter(input_host_path: str, input_ext: str, quality: str, trim_value: Optional[str], is_audio: bool) -> Tuple[str, dict]:
|
||||||
rid = __import__('uuid').uuid4().hex[:8]
|
rid = __import__('uuid').uuid4().hex[:8]
|
||||||
output_dir_container = f"/tmp/conv_{rid}"
|
output_dir_container = f"/tmp/conv_{rid}"
|
||||||
output_dir_host = f"/tmp/conv_{rid}"
|
output_dir_host = f"/tmp/conv_{rid}"
|
||||||
|
|
@ -215,7 +215,7 @@ async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
|
||||||
if required.issubset(kinds_ready):
|
if required.issubset(kinds_ready):
|
||||||
continue
|
continue
|
||||||
# Always decrypt from IPFS using local or remote key
|
# Always decrypt from IPFS using local or remote key
|
||||||
storage_path: str | None = None
|
storage_path: Optional[str] = None
|
||||||
ck = (await session.execute(select(ContentKey).where(ContentKey.content_id == ec.id))).scalars().first()
|
ck = (await session.execute(select(ContentKey).where(ContentKey.content_id == ec.id))).scalars().first()
|
||||||
if ck:
|
if ck:
|
||||||
storage_path = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
|
storage_path = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
|
||||||
|
|
@ -285,7 +285,7 @@ async def main_fn(memory):
|
||||||
await worker_loop()
|
await worker_loop()
|
||||||
|
|
||||||
|
|
||||||
async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> str | None:
|
async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> Optional[str]:
|
||||||
"""Download encrypted ENCF stream from IPFS and decrypt on the fly into a temp file."""
|
"""Download encrypted ENCF stream from IPFS and decrypt on the fly into a temp file."""
|
||||||
import tempfile
|
import tempfile
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,6 @@ import os
|
||||||
import struct
|
import struct
|
||||||
from typing import BinaryIO, Iterator, AsyncIterator
|
from typing import BinaryIO, Iterator, AsyncIterator
|
||||||
|
|
||||||
from Crypto.Cipher import SIV
|
|
||||||
from Crypto.Cipher import AES
|
from Crypto.Cipher import AES
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -62,7 +61,7 @@ def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: byte
|
||||||
block = src.read(chunk_bytes)
|
block = src.read(chunk_bytes)
|
||||||
if not block:
|
if not block:
|
||||||
break
|
break
|
||||||
siv = SIV.new(key=key, ciphermod=AES) # new object per message
|
siv = AES.new(key, AES.MODE_SIV) # new object per message
|
||||||
siv.update(_ad(salt, idx))
|
siv.update(_ad(salt, idx))
|
||||||
ciph, tag = siv.encrypt_and_digest(block)
|
ciph, tag = siv.encrypt_and_digest(block)
|
||||||
yield struct.pack(">I", len(block))
|
yield struct.pack(">I", len(block))
|
||||||
|
|
@ -76,7 +75,6 @@ async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_
|
||||||
Parse ENCF v1 stream from async byte iterator and write plaintext to out_path.
|
Parse ENCF v1 stream from async byte iterator and write plaintext to out_path.
|
||||||
"""
|
"""
|
||||||
import aiofiles
|
import aiofiles
|
||||||
from Crypto.Cipher import SIV as _SIV
|
|
||||||
from Crypto.Cipher import AES as _AES
|
from Crypto.Cipher import AES as _AES
|
||||||
|
|
||||||
buf = bytearray()
|
buf = bytearray()
|
||||||
|
|
@ -129,7 +127,7 @@ async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_
|
||||||
c = bytes(buf[:p_len])
|
c = bytes(buf[:p_len])
|
||||||
t = bytes(buf[p_len:p_len+TAG_LEN])
|
t = bytes(buf[p_len:p_len+TAG_LEN])
|
||||||
del buf[:p_len+TAG_LEN]
|
del buf[:p_len+TAG_LEN]
|
||||||
siv = _SIV.new(key=key, ciphermod=_AES)
|
siv = _AES.new(key, _AES.MODE_SIV)
|
||||||
siv.update(_ad(salt, idx))
|
siv.update(_ad(salt, idx))
|
||||||
p = siv.decrypt_and_verify(c, t)
|
p = siv.decrypt_and_verify(c, t)
|
||||||
await out.write(p)
|
await out.write(p)
|
||||||
|
|
|
||||||
|
|
@ -26,8 +26,19 @@ async def add_streamed_file(stream_iter: Iterable[bytes], filename: str = "file.
|
||||||
}
|
}
|
||||||
q = {**default_params, **params}
|
q = {**default_params, **params}
|
||||||
|
|
||||||
|
class _StreamAdapter:
|
||||||
|
def __init__(self, iterable):
|
||||||
|
self._iter = iter(iterable)
|
||||||
|
|
||||||
|
def read(self, size=-1):
|
||||||
|
try:
|
||||||
|
return next(self._iter)
|
||||||
|
except StopIteration:
|
||||||
|
return b''
|
||||||
|
|
||||||
|
stream = _StreamAdapter(stream_iter)
|
||||||
async with httpx.AsyncClient(timeout=None) as client:
|
async with httpx.AsyncClient(timeout=None) as client:
|
||||||
files = {"file": (filename, stream_iter, "application/octet-stream")}
|
files = {"file": (filename, stream, "application/octet-stream")}
|
||||||
r = await client.post(f"{IPFS_API_URL}/api/v0/add", params=q, files=files)
|
r = await client.post(f"{IPFS_API_URL}/api/v0/add", params=q, files=files)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
# /add may emit NDJSON lines; most often single JSON
|
# /add may emit NDJSON lines; most often single JSON
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue