From bb2180f44bd2caffb291c05ad839bbbec11e6ea3 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 28 Feb 2025 17:30:39 +0300 Subject: [PATCH] fixes --- app/api/middleware.py | 2 +- app/api/routes/content.py | 2 - app/api/routes/progressive_storage.py | 70 ++++-------- uploader_test.html | 146 +++++++++----------------- 4 files changed, 74 insertions(+), 146 deletions(-) diff --git a/app/api/middleware.py b/app/api/middleware.py index 4837e85..f68db6b 100644 --- a/app/api/middleware.py +++ b/app/api/middleware.py @@ -15,7 +15,7 @@ from datetime import datetime, timedelta def attach_headers(response): response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS" - response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-content-sha256, x-chunk-start, x-upload-id" + response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id" response.headers["Access-Control-Allow-Credentials"] = "true" return response diff --git a/app/api/routes/content.py b/app/api/routes/content.py index f66c475..81520c6 100644 --- a/app/api/routes/content.py +++ b/app/api/routes/content.py @@ -147,8 +147,6 @@ async def s_api_v1_content_view(request, content_address: str): async def s_api_v1_content_friendly_list(request): # return html table with content list. bootstrap is used - - result = """ diff --git a/app/api/routes/progressive_storage.py b/app/api/routes/progressive_storage.py index db9c731..4cf7333 100644 --- a/app/api/routes/progressive_storage.py +++ b/app/api/routes/progressive_storage.py @@ -21,33 +21,6 @@ async def s_api_v1_5_storage_post(request): # Log the receipt of a chunk upload request make_log("uploader_v1.5", "Received chunk upload request", level="INFO") - # Get the provided file hash from header (hex format) - provided_hash_hex = request.headers.get("X-Content-SHA256") - if not provided_hash_hex: - make_log("uploader_v1.5", "Missing X-Content-SHA256 header", level="ERROR") - return response.json({"error": "Missing X-Content-SHA256 header"}, status=400) - try: - provided_hash_bytes = bytes.fromhex(provided_hash_hex) - provided_hash_b58 = b58encode(provided_hash_bytes).decode() - make_log("uploader_v1.5", f"Provided hash (base58): {provided_hash_b58}", level="INFO") - except Exception as e: - make_log("uploader_v1.5", f"Invalid X-Content-SHA256 header format: {e}", level="ERROR") - return response.json({"error": "Invalid X-Content-SHA256 header format"}, status=400) - - existing = request.ctx.db_session.query(StoredContent).filter_by(hash=provided_hash_b58).first() - if existing: - make_log("uploader_v1.5", f"File with hash {provided_hash_b58} already exists in DB", level="INFO") - serialized_v2 = existing.cid.serialize_v2() # Get v2 content id - serialized_v1 = existing.cid.serialize_v1() # Get v1 content id - # Return early response since the file is already stored - return response.json({ - "upload_id": request.headers.get("X-Upload-ID", str(uuid4())), # Use provided or generate a new upload_id - "content_sha256": provided_hash_b58, - "content_id": serialized_v2, - "content_id_v1": serialized_v1, - "content_url": f"dmy://storage?cid={serialized_v2}", - }) - # Get the provided file name from header and decode it from base64 provided_filename_b64 = request.headers.get("X-File-Name") if not provided_filename_b64: @@ -125,28 +98,29 @@ async def s_api_v1_5_storage_post(request): make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR") return response.json({"error": "Failed to save chunk"}, status=500) - # Compute the SHA256 hash of the temporary file using subprocess - try: - proc = await asyncio.create_subprocess_exec( - 'sha256sum', temp_path, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - if proc.returncode != 0: - error_msg = stderr.decode().strip() - make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR") - return response.json({"error": "Failed to compute file hash"}, status=500) - computed_hash_hex = stdout.decode().split()[0].strip() - computed_hash_bytes = bytes.fromhex(computed_hash_hex) - computed_hash_b58 = b58encode(computed_hash_bytes).decode() - make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO") - except Exception as e: - make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR") - return response.json({"error": "Error computing file hash"}, status=500) - # If computed hash matches the provided one, the final chunk has been received - if computed_hash_b58 == provided_hash_b58: + is_last_chunk = int(request.headers.get("X-Last-Chunk", "0")) == 1 + if is_last_chunk: + # Compute the SHA256 hash of the temporary file using subprocess + try: + proc = await asyncio.create_subprocess_exec( + 'sha256sum', temp_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + error_msg = stderr.decode().strip() + make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR") + return response.json({"error": "Failed to compute file hash"}, status=500) + computed_hash_hex = stdout.decode().split()[0].strip() + computed_hash_bytes = bytes.fromhex(computed_hash_hex) + computed_hash_b58 = b58encode(computed_hash_bytes).decode() + make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO") + except Exception as e: + make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR") + return response.json({"error": "Error computing file hash"}, status=500) + final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}") try: os.rename(temp_path, final_path) diff --git a/uploader_test.html b/uploader_test.html index d4e192a..438cbe4 100644 --- a/uploader_test.html +++ b/uploader_test.html @@ -10,8 +10,6 @@ input, button { margin-bottom: 10px; } #log { border: 1px solid #ccc; padding: 10px; max-height: 200px; overflow-y: auto; background: #f9f9f9; } - -

Загрузка и стриминг файла

@@ -51,105 +49,63 @@ logDiv.appendChild(p); } - // Compute SHA-256 hash of a file using jsSHA library (incremental reading) - function computeSHA256(file) { - return new Promise((resolve, reject) => { - const chunkSize = 2097152 * 10; // 2MB per chunk - let offset = 0; - const reader = new FileReader(); - const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER"); - - reader.onload = function(e) { - // Update hash with current chunk data - shaObj.update(e.target.result); - offset += chunkSize; - appendLog(`Processed ${Math.min(offset, file.size)} из ${file.size} байт`); - if (offset < file.size) { - readNextChunk(); - } else { - try { - const hash = shaObj.getHash("HEX"); - resolve(hash); - } catch (err) { - reject(err); - } - } - }; - - reader.onerror = function(err) { - reject(err); - }; - - function readNextChunk() { - const slice = file.slice(offset, offset + chunkSize); - reader.readAsArrayBuffer(slice); - } - - readNextChunk(); - }); - } - - // Upload file in chunks (max 80 MB per chunk) + // Upload file in chunks (max 80 MB per chunk) without computing file hash async function uploadFileInChunks(file) { const maxChunkSize = 80 * 1024 * 1024; // 80 MB let offset = 0; let uploadId = null; - try { - appendLog("Starting hash computation..."); - const hashHex = await computeSHA256(file); - appendLog(`Computed SHA-256 hash: ${hashHex}`); + appendLog("Starting file upload..."); + + while (offset < file.size) { + const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size)); + appendLog(`Uploading chunk starting at byte ${offset}`); - while (offset < file.size) { - const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size)); - appendLog(`Uploading chunk starting at byte ${offset}`); - - // Prepare headers for the chunk upload - const headers = { - "X-Content-SHA256": hashHex, - "X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64 - "X-Chunk-Start": offset.toString(), - "Content-Type": file.type || "application/octet-stream" - }; - if (uploadId) { - headers["X-Upload-ID"] = uploadId; - } - - const response = await fetch(BASE_URL, { - method: "POST", - headers: headers, - body: chunk - }); - - if (!response.ok) { - const errorData = await response.json(); - appendLog(`Chunk upload failed: ${errorData.error}`); - throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`); - } - - const resultData = await response.json(); - // Save uploadId from first response if not set - if (!uploadId && resultData.upload_id) { - uploadId = resultData.upload_id; - } - - // If final response contains content_id, upload is complete - if (resultData.content_id) { - appendLog(`Upload complete. File ID: ${resultData.content_id}`); - return resultData; - } - - // Update offset based on server-reported current size - if (resultData.current_size !== undefined) { - offset = resultData.current_size; - appendLog(`Server reports current_size: ${offset}`); - } else { - appendLog("Unexpected response from server, missing current_size."); - throw new Error("Missing current_size in response"); - } + // Prepare headers for the chunk upload + const headers = { + "X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64 + "X-Chunk-Start": offset.toString(), + "Content-Type": file.type || "application/octet-stream" + }; + if (uploadId) { + headers["X-Upload-ID"] = uploadId; + } + // Set header to indicate the last chunk if this is the final part of the file + if (offset + chunk.size >= file.size) { + headers["X-Last-Chunk"] = "1"; + } + + const response = await fetch(BASE_URL, { + method: "POST", + headers: headers, + body: chunk + }); + + if (!response.ok) { + const errorData = await response.json(); + appendLog(`Chunk upload failed: ${errorData.error}`); + throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`); + } + + const resultData = await response.json(); + // Save uploadId from first response if not set + if (!uploadId && resultData.upload_id) { + uploadId = resultData.upload_id; + } + + // If final response contains content_id, upload is complete + if (resultData.content_id) { + appendLog(`Upload complete. File ID: ${resultData.content_id}`); + return resultData; + } + + // Update offset based on server-reported current size + if (resultData.current_size !== undefined) { + offset = resultData.current_size; + appendLog(`Server reports current_size: ${offset}`); + } else { + appendLog("Unexpected response from server, missing current_size."); + throw new Error("Missing current_size in response"); } - } catch (err) { - appendLog(`Error during upload: ${err}`); - throw err; } }