fix get storage streaming

This commit is contained in:
user 2025-02-25 17:39:27 +03:00
parent 7d6b1018ad
commit 080f000fd2
2 changed files with 44 additions and 46 deletions

View File

@ -190,7 +190,6 @@ async def s_api_v1_5_storage_post(request):
# GET /api/v1.5/storage/<file_hash> # GET /api/v1.5/storage/<file_hash>
async def s_api_v1_5_storage_get(request, file_hash): async def s_api_v1_5_storage_get(request, file_hash):
# Log the file retrieval request
make_log("uploader_v1.5", f"Received file retrieval request for hash: {file_hash}", level="INFO") make_log("uploader_v1.5", f"Received file retrieval request for hash: {file_hash}", level="INFO")
final_path = os.path.join(UPLOADS_DIR, f"v1.5_{file_hash}") final_path = os.path.join(UPLOADS_DIR, f"v1.5_{file_hash}")
@ -228,17 +227,11 @@ async def s_api_v1_5_storage_get(request, file_hash):
start_str, end_str = part.split('-', 1) start_str, end_str = part.split('-', 1)
if start_str == "": if start_str == "":
suffix_length = int(end_str) suffix_length = int(end_str)
if suffix_length > file_size: start = 0 if suffix_length > file_size else file_size - suffix_length
start = 0
else:
start = file_size - suffix_length
end = file_size - 1 end = file_size - 1
else: else:
start = int(start_str) start = int(start_str)
if end_str == "": end = file_size - 1 if end_str == "" else int(end_str)
end = file_size - 1
else:
end = int(end_str)
if start > end or end >= file_size: if start > end or end >= file_size:
raise ValueError("Requested Range Not Satisfiable") raise ValueError("Requested Range Not Satisfiable")
parsed_ranges.append((start, end)) parsed_ranges.append((start, end))
@ -247,6 +240,7 @@ async def s_api_v1_5_storage_get(request, file_hash):
return response.json({"error": "Invalid Range header"}, status=400) return response.json({"error": "Invalid Range header"}, status=400)
if len(parsed_ranges) == 1: if len(parsed_ranges) == 1:
# Single range streaming
start, end = parsed_ranges[0] start, end = parsed_ranges[0]
content_length = end - start + 1 content_length = end - start + 1
headers = { headers = {
@ -255,28 +249,31 @@ async def s_api_v1_5_storage_get(request, file_hash):
"Content-Length": str(content_length), "Content-Length": str(content_length),
"Content-Type": mime_type, "Content-Type": mime_type,
} }
async def stream_file_range(): # Create response for streaming
stream_response = await request.respond(headers=headers, status=206, content_type=mime_type)
make_log("uploader_v1.5", f"Starting to stream file from byte {start} to {end}", level="INFO") make_log("uploader_v1.5", f"Starting to stream file from byte {start} to {end}", level="INFO")
async with aiofiles.open(final_path, mode='rb') as f: async with aiofiles.open(final_path, mode='rb') as f:
await f.seek(start) await f.seek(start)
remaining = content_length remaining = content_length
chunk_size = 1024 * 1024 chunk_size = 1024 * 1024 # chunk size in bytes
while remaining > 0: while remaining > 0:
read_size = min(chunk_size, remaining) read_size = min(chunk_size, remaining)
data = await f.read(read_size) data = await f.read(read_size)
if not data: if not data:
break break
remaining -= len(data) remaining -= len(data)
yield data await stream_response.send(data)
make_log("uploader_v1.5", f"Finished streaming file: {final_path}", level="INFO") make_log("uploader_v1.5", f"Finished streaming file: {final_path}", level="INFO")
return response.stream(stream_file_range, status=206, headers=headers) await stream_response.eof()
return stream_response
else: else:
# Multipart range streaming
boundary = uuid4().hex boundary = uuid4().hex
headers = { headers = {
"Content-Type": f"multipart/byteranges; boundary={boundary}", "Content-Type": f"multipart/byteranges; boundary={boundary}",
"Accept-Ranges": "bytes", "Accept-Ranges": "bytes",
} }
async def stream_multipart(): stream_response = await request.respond(headers=headers, status=206)
for start, end in parsed_ranges: for start, end in parsed_ranges:
part_header = ( part_header = (
f"--{boundary}\r\n" f"--{boundary}\r\n"
@ -284,7 +281,7 @@ async def s_api_v1_5_storage_get(request, file_hash):
f"Content-Range: bytes {start}-{end}/{file_size}\r\n" f"Content-Range: bytes {start}-{end}/{file_size}\r\n"
f"\r\n" f"\r\n"
) )
yield part_header.encode() await stream_response.send(part_header.encode())
part_length = end - start + 1 part_length = end - start + 1
async with aiofiles.open(final_path, mode='rb') as f: async with aiofiles.open(final_path, mode='rb') as f:
await f.seek(start) await f.seek(start)
@ -296,10 +293,11 @@ async def s_api_v1_5_storage_get(request, file_hash):
if not data: if not data:
break break
remaining -= len(data) remaining -= len(data)
yield data await stream_response.send(data)
yield b"\r\n" await stream_response.send(b"\r\n")
yield f"--{boundary}--\r\n".encode() await stream_response.send(f"--{boundary}--\r\n".encode())
return response.stream(stream_multipart, status=206, headers=headers) await stream_response.eof()
return stream_response
else: else:
make_log("uploader_v1.5", f"Returning full file for video/audio: {final_path}", level="INFO") make_log("uploader_v1.5", f"Returning full file for video/audio: {final_path}", level="INFO")
return await response.file(final_path, mime_type=mime_type) return await response.file(final_path, mime_type=mime_type)

View File

@ -54,7 +54,7 @@
// Compute SHA-256 hash of a file using jsSHA library (incremental reading) // Compute SHA-256 hash of a file using jsSHA library (incremental reading)
function computeSHA256(file) { function computeSHA256(file) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const chunkSize = 2097152; // 2MB per chunk const chunkSize = 2097152 * 10; // 2MB per chunk
let offset = 0; let offset = 0;
const reader = new FileReader(); const reader = new FileReader();
const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER"); const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");