fix get storage streaming
This commit is contained in:
parent
7d6b1018ad
commit
080f000fd2
|
|
@ -190,7 +190,6 @@ async def s_api_v1_5_storage_post(request):
|
||||||
|
|
||||||
# GET /api/v1.5/storage/<file_hash>
|
# GET /api/v1.5/storage/<file_hash>
|
||||||
async def s_api_v1_5_storage_get(request, file_hash):
|
async def s_api_v1_5_storage_get(request, file_hash):
|
||||||
# Log the file retrieval request
|
|
||||||
make_log("uploader_v1.5", f"Received file retrieval request for hash: {file_hash}", level="INFO")
|
make_log("uploader_v1.5", f"Received file retrieval request for hash: {file_hash}", level="INFO")
|
||||||
|
|
||||||
final_path = os.path.join(UPLOADS_DIR, f"v1.5_{file_hash}")
|
final_path = os.path.join(UPLOADS_DIR, f"v1.5_{file_hash}")
|
||||||
|
|
@ -228,17 +227,11 @@ async def s_api_v1_5_storage_get(request, file_hash):
|
||||||
start_str, end_str = part.split('-', 1)
|
start_str, end_str = part.split('-', 1)
|
||||||
if start_str == "":
|
if start_str == "":
|
||||||
suffix_length = int(end_str)
|
suffix_length = int(end_str)
|
||||||
if suffix_length > file_size:
|
start = 0 if suffix_length > file_size else file_size - suffix_length
|
||||||
start = 0
|
|
||||||
else:
|
|
||||||
start = file_size - suffix_length
|
|
||||||
end = file_size - 1
|
end = file_size - 1
|
||||||
else:
|
else:
|
||||||
start = int(start_str)
|
start = int(start_str)
|
||||||
if end_str == "":
|
end = file_size - 1 if end_str == "" else int(end_str)
|
||||||
end = file_size - 1
|
|
||||||
else:
|
|
||||||
end = int(end_str)
|
|
||||||
if start > end or end >= file_size:
|
if start > end or end >= file_size:
|
||||||
raise ValueError("Requested Range Not Satisfiable")
|
raise ValueError("Requested Range Not Satisfiable")
|
||||||
parsed_ranges.append((start, end))
|
parsed_ranges.append((start, end))
|
||||||
|
|
@ -247,6 +240,7 @@ async def s_api_v1_5_storage_get(request, file_hash):
|
||||||
return response.json({"error": "Invalid Range header"}, status=400)
|
return response.json({"error": "Invalid Range header"}, status=400)
|
||||||
|
|
||||||
if len(parsed_ranges) == 1:
|
if len(parsed_ranges) == 1:
|
||||||
|
# Single range streaming
|
||||||
start, end = parsed_ranges[0]
|
start, end = parsed_ranges[0]
|
||||||
content_length = end - start + 1
|
content_length = end - start + 1
|
||||||
headers = {
|
headers = {
|
||||||
|
|
@ -255,28 +249,31 @@ async def s_api_v1_5_storage_get(request, file_hash):
|
||||||
"Content-Length": str(content_length),
|
"Content-Length": str(content_length),
|
||||||
"Content-Type": mime_type,
|
"Content-Type": mime_type,
|
||||||
}
|
}
|
||||||
async def stream_file_range():
|
# Create response for streaming
|
||||||
|
stream_response = await request.respond(headers=headers, status=206, content_type=mime_type)
|
||||||
make_log("uploader_v1.5", f"Starting to stream file from byte {start} to {end}", level="INFO")
|
make_log("uploader_v1.5", f"Starting to stream file from byte {start} to {end}", level="INFO")
|
||||||
async with aiofiles.open(final_path, mode='rb') as f:
|
async with aiofiles.open(final_path, mode='rb') as f:
|
||||||
await f.seek(start)
|
await f.seek(start)
|
||||||
remaining = content_length
|
remaining = content_length
|
||||||
chunk_size = 1024 * 1024
|
chunk_size = 1024 * 1024 # chunk size in bytes
|
||||||
while remaining > 0:
|
while remaining > 0:
|
||||||
read_size = min(chunk_size, remaining)
|
read_size = min(chunk_size, remaining)
|
||||||
data = await f.read(read_size)
|
data = await f.read(read_size)
|
||||||
if not data:
|
if not data:
|
||||||
break
|
break
|
||||||
remaining -= len(data)
|
remaining -= len(data)
|
||||||
yield data
|
await stream_response.send(data)
|
||||||
make_log("uploader_v1.5", f"Finished streaming file: {final_path}", level="INFO")
|
make_log("uploader_v1.5", f"Finished streaming file: {final_path}", level="INFO")
|
||||||
return response.stream(stream_file_range, status=206, headers=headers)
|
await stream_response.eof()
|
||||||
|
return stream_response
|
||||||
else:
|
else:
|
||||||
|
# Multipart range streaming
|
||||||
boundary = uuid4().hex
|
boundary = uuid4().hex
|
||||||
headers = {
|
headers = {
|
||||||
"Content-Type": f"multipart/byteranges; boundary={boundary}",
|
"Content-Type": f"multipart/byteranges; boundary={boundary}",
|
||||||
"Accept-Ranges": "bytes",
|
"Accept-Ranges": "bytes",
|
||||||
}
|
}
|
||||||
async def stream_multipart():
|
stream_response = await request.respond(headers=headers, status=206)
|
||||||
for start, end in parsed_ranges:
|
for start, end in parsed_ranges:
|
||||||
part_header = (
|
part_header = (
|
||||||
f"--{boundary}\r\n"
|
f"--{boundary}\r\n"
|
||||||
|
|
@ -284,7 +281,7 @@ async def s_api_v1_5_storage_get(request, file_hash):
|
||||||
f"Content-Range: bytes {start}-{end}/{file_size}\r\n"
|
f"Content-Range: bytes {start}-{end}/{file_size}\r\n"
|
||||||
f"\r\n"
|
f"\r\n"
|
||||||
)
|
)
|
||||||
yield part_header.encode()
|
await stream_response.send(part_header.encode())
|
||||||
part_length = end - start + 1
|
part_length = end - start + 1
|
||||||
async with aiofiles.open(final_path, mode='rb') as f:
|
async with aiofiles.open(final_path, mode='rb') as f:
|
||||||
await f.seek(start)
|
await f.seek(start)
|
||||||
|
|
@ -296,10 +293,11 @@ async def s_api_v1_5_storage_get(request, file_hash):
|
||||||
if not data:
|
if not data:
|
||||||
break
|
break
|
||||||
remaining -= len(data)
|
remaining -= len(data)
|
||||||
yield data
|
await stream_response.send(data)
|
||||||
yield b"\r\n"
|
await stream_response.send(b"\r\n")
|
||||||
yield f"--{boundary}--\r\n".encode()
|
await stream_response.send(f"--{boundary}--\r\n".encode())
|
||||||
return response.stream(stream_multipart, status=206, headers=headers)
|
await stream_response.eof()
|
||||||
|
return stream_response
|
||||||
else:
|
else:
|
||||||
make_log("uploader_v1.5", f"Returning full file for video/audio: {final_path}", level="INFO")
|
make_log("uploader_v1.5", f"Returning full file for video/audio: {final_path}", level="INFO")
|
||||||
return await response.file(final_path, mime_type=mime_type)
|
return await response.file(final_path, mime_type=mime_type)
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@
|
||||||
// Compute SHA-256 hash of a file using jsSHA library (incremental reading)
|
// Compute SHA-256 hash of a file using jsSHA library (incremental reading)
|
||||||
function computeSHA256(file) {
|
function computeSHA256(file) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const chunkSize = 2097152; // 2MB per chunk
|
const chunkSize = 2097152 * 10; // 2MB per chunk
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
const reader = new FileReader();
|
const reader = new FileReader();
|
||||||
const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");
|
const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue