From 080f000fd298454cf6c31c2fbf2f979d9e044317 Mon Sep 17 00:00:00 2001 From: user Date: Tue, 25 Feb 2025 17:39:27 +0300 Subject: [PATCH] fix get storage streaming --- app/api/routes/progressive_storage.py | 88 +++++++++++++-------------- uploader_test.html | 2 +- 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/app/api/routes/progressive_storage.py b/app/api/routes/progressive_storage.py index f3f93c0..d490743 100644 --- a/app/api/routes/progressive_storage.py +++ b/app/api/routes/progressive_storage.py @@ -190,7 +190,6 @@ async def s_api_v1_5_storage_post(request): # GET /api/v1.5/storage/ async def s_api_v1_5_storage_get(request, file_hash): - # Log the file retrieval request make_log("uploader_v1.5", f"Received file retrieval request for hash: {file_hash}", level="INFO") final_path = os.path.join(UPLOADS_DIR, f"v1.5_{file_hash}") @@ -228,17 +227,11 @@ async def s_api_v1_5_storage_get(request, file_hash): start_str, end_str = part.split('-', 1) if start_str == "": suffix_length = int(end_str) - if suffix_length > file_size: - start = 0 - else: - start = file_size - suffix_length + start = 0 if suffix_length > file_size else file_size - suffix_length end = file_size - 1 else: start = int(start_str) - if end_str == "": - end = file_size - 1 - else: - end = int(end_str) + end = file_size - 1 if end_str == "" else int(end_str) if start > end or end >= file_size: raise ValueError("Requested Range Not Satisfiable") parsed_ranges.append((start, end)) @@ -247,6 +240,7 @@ async def s_api_v1_5_storage_get(request, file_hash): return response.json({"error": "Invalid Range header"}, status=400) if len(parsed_ranges) == 1: + # Single range streaming start, end = parsed_ranges[0] content_length = end - start + 1 headers = { @@ -255,11 +249,43 @@ async def s_api_v1_5_storage_get(request, file_hash): "Content-Length": str(content_length), "Content-Type": mime_type, } - async def stream_file_range(): - make_log("uploader_v1.5", f"Starting to stream file from byte {start} to {end}", level="INFO") + # Create response for streaming + stream_response = await request.respond(headers=headers, status=206, content_type=mime_type) + make_log("uploader_v1.5", f"Starting to stream file from byte {start} to {end}", level="INFO") + async with aiofiles.open(final_path, mode='rb') as f: + await f.seek(start) + remaining = content_length + chunk_size = 1024 * 1024 # chunk size in bytes + while remaining > 0: + read_size = min(chunk_size, remaining) + data = await f.read(read_size) + if not data: + break + remaining -= len(data) + await stream_response.send(data) + make_log("uploader_v1.5", f"Finished streaming file: {final_path}", level="INFO") + await stream_response.eof() + return stream_response + else: + # Multipart range streaming + boundary = uuid4().hex + headers = { + "Content-Type": f"multipart/byteranges; boundary={boundary}", + "Accept-Ranges": "bytes", + } + stream_response = await request.respond(headers=headers, status=206) + for start, end in parsed_ranges: + part_header = ( + f"--{boundary}\r\n" + f"Content-Type: {mime_type}\r\n" + f"Content-Range: bytes {start}-{end}/{file_size}\r\n" + f"\r\n" + ) + await stream_response.send(part_header.encode()) + part_length = end - start + 1 async with aiofiles.open(final_path, mode='rb') as f: await f.seek(start) - remaining = content_length + remaining = part_length chunk_size = 1024 * 1024 while remaining > 0: read_size = min(chunk_size, remaining) @@ -267,39 +293,11 @@ async def s_api_v1_5_storage_get(request, file_hash): if not data: break remaining -= len(data) - yield data - make_log("uploader_v1.5", f"Finished streaming file: {final_path}", level="INFO") - return response.stream(stream_file_range, status=206, headers=headers) - else: - boundary = uuid4().hex - headers = { - "Content-Type": f"multipart/byteranges; boundary={boundary}", - "Accept-Ranges": "bytes", - } - async def stream_multipart(): - for start, end in parsed_ranges: - part_header = ( - f"--{boundary}\r\n" - f"Content-Type: {mime_type}\r\n" - f"Content-Range: bytes {start}-{end}/{file_size}\r\n" - f"\r\n" - ) - yield part_header.encode() - part_length = end - start + 1 - async with aiofiles.open(final_path, mode='rb') as f: - await f.seek(start) - remaining = part_length - chunk_size = 1024 * 1024 - while remaining > 0: - read_size = min(chunk_size, remaining) - data = await f.read(read_size) - if not data: - break - remaining -= len(data) - yield data - yield b"\r\n" - yield f"--{boundary}--\r\n".encode() - return response.stream(stream_multipart, status=206, headers=headers) + await stream_response.send(data) + await stream_response.send(b"\r\n") + await stream_response.send(f"--{boundary}--\r\n".encode()) + await stream_response.eof() + return stream_response else: make_log("uploader_v1.5", f"Returning full file for video/audio: {final_path}", level="INFO") return await response.file(final_path, mime_type=mime_type) diff --git a/uploader_test.html b/uploader_test.html index 87b5bb6..d4e192a 100644 --- a/uploader_test.html +++ b/uploader_test.html @@ -54,7 +54,7 @@ // Compute SHA-256 hash of a file using jsSHA library (incremental reading) function computeSHA256(file) { return new Promise((resolve, reject) => { - const chunkSize = 2097152; // 2MB per chunk + const chunkSize = 2097152 * 10; // 2MB per chunk let offset = 0; const reader = new FileReader(); const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");