diff --git a/app/api/middleware.py b/app/api/middleware.py
index 4837e85..f68db6b 100644
--- a/app/api/middleware.py
+++ b/app/api/middleware.py
@@ -15,7 +15,7 @@ from datetime import datetime, timedelta
def attach_headers(response):
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
- response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-content-sha256, x-chunk-start, x-upload-id"
+ response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
response.headers["Access-Control-Allow-Credentials"] = "true"
return response
diff --git a/app/api/routes/content.py b/app/api/routes/content.py
index f66c475..81520c6 100644
--- a/app/api/routes/content.py
+++ b/app/api/routes/content.py
@@ -147,8 +147,6 @@ async def s_api_v1_content_view(request, content_address: str):
async def s_api_v1_content_friendly_list(request):
# return html table with content list. bootstrap is used
-
-
result = """
diff --git a/app/api/routes/progressive_storage.py b/app/api/routes/progressive_storage.py
index db9c731..4cf7333 100644
--- a/app/api/routes/progressive_storage.py
+++ b/app/api/routes/progressive_storage.py
@@ -21,33 +21,6 @@ async def s_api_v1_5_storage_post(request):
# Log the receipt of a chunk upload request
make_log("uploader_v1.5", "Received chunk upload request", level="INFO")
- # Get the provided file hash from header (hex format)
- provided_hash_hex = request.headers.get("X-Content-SHA256")
- if not provided_hash_hex:
- make_log("uploader_v1.5", "Missing X-Content-SHA256 header", level="ERROR")
- return response.json({"error": "Missing X-Content-SHA256 header"}, status=400)
- try:
- provided_hash_bytes = bytes.fromhex(provided_hash_hex)
- provided_hash_b58 = b58encode(provided_hash_bytes).decode()
- make_log("uploader_v1.5", f"Provided hash (base58): {provided_hash_b58}", level="INFO")
- except Exception as e:
- make_log("uploader_v1.5", f"Invalid X-Content-SHA256 header format: {e}", level="ERROR")
- return response.json({"error": "Invalid X-Content-SHA256 header format"}, status=400)
-
- existing = request.ctx.db_session.query(StoredContent).filter_by(hash=provided_hash_b58).first()
- if existing:
- make_log("uploader_v1.5", f"File with hash {provided_hash_b58} already exists in DB", level="INFO")
- serialized_v2 = existing.cid.serialize_v2() # Get v2 content id
- serialized_v1 = existing.cid.serialize_v1() # Get v1 content id
- # Return early response since the file is already stored
- return response.json({
- "upload_id": request.headers.get("X-Upload-ID", str(uuid4())), # Use provided or generate a new upload_id
- "content_sha256": provided_hash_b58,
- "content_id": serialized_v2,
- "content_id_v1": serialized_v1,
- "content_url": f"dmy://storage?cid={serialized_v2}",
- })
-
# Get the provided file name from header and decode it from base64
provided_filename_b64 = request.headers.get("X-File-Name")
if not provided_filename_b64:
@@ -125,28 +98,29 @@ async def s_api_v1_5_storage_post(request):
make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR")
return response.json({"error": "Failed to save chunk"}, status=500)
- # Compute the SHA256 hash of the temporary file using subprocess
- try:
- proc = await asyncio.create_subprocess_exec(
- 'sha256sum', temp_path,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE
- )
- stdout, stderr = await proc.communicate()
- if proc.returncode != 0:
- error_msg = stderr.decode().strip()
- make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
- return response.json({"error": "Failed to compute file hash"}, status=500)
- computed_hash_hex = stdout.decode().split()[0].strip()
- computed_hash_bytes = bytes.fromhex(computed_hash_hex)
- computed_hash_b58 = b58encode(computed_hash_bytes).decode()
- make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
- except Exception as e:
- make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
- return response.json({"error": "Error computing file hash"}, status=500)
-
# If computed hash matches the provided one, the final chunk has been received
- if computed_hash_b58 == provided_hash_b58:
+ is_last_chunk = int(request.headers.get("X-Last-Chunk", "0")) == 1
+ if is_last_chunk:
+ # Compute the SHA256 hash of the temporary file using subprocess
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ 'sha256sum', temp_path,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE
+ )
+ stdout, stderr = await proc.communicate()
+ if proc.returncode != 0:
+ error_msg = stderr.decode().strip()
+ make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
+ return response.json({"error": "Failed to compute file hash"}, status=500)
+ computed_hash_hex = stdout.decode().split()[0].strip()
+ computed_hash_bytes = bytes.fromhex(computed_hash_hex)
+ computed_hash_b58 = b58encode(computed_hash_bytes).decode()
+ make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
+ except Exception as e:
+ make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
+ return response.json({"error": "Error computing file hash"}, status=500)
+
final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}")
try:
os.rename(temp_path, final_path)
diff --git a/uploader_test.html b/uploader_test.html
index d4e192a..438cbe4 100644
--- a/uploader_test.html
+++ b/uploader_test.html
@@ -10,8 +10,6 @@
input, button { margin-bottom: 10px; }
#log { border: 1px solid #ccc; padding: 10px; max-height: 200px; overflow-y: auto; background: #f9f9f9; }
-
-
Загрузка и стриминг файла
@@ -51,105 +49,63 @@
logDiv.appendChild(p);
}
- // Compute SHA-256 hash of a file using jsSHA library (incremental reading)
- function computeSHA256(file) {
- return new Promise((resolve, reject) => {
- const chunkSize = 2097152 * 10; // 2MB per chunk
- let offset = 0;
- const reader = new FileReader();
- const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");
-
- reader.onload = function(e) {
- // Update hash with current chunk data
- shaObj.update(e.target.result);
- offset += chunkSize;
- appendLog(`Processed ${Math.min(offset, file.size)} из ${file.size} байт`);
- if (offset < file.size) {
- readNextChunk();
- } else {
- try {
- const hash = shaObj.getHash("HEX");
- resolve(hash);
- } catch (err) {
- reject(err);
- }
- }
- };
-
- reader.onerror = function(err) {
- reject(err);
- };
-
- function readNextChunk() {
- const slice = file.slice(offset, offset + chunkSize);
- reader.readAsArrayBuffer(slice);
- }
-
- readNextChunk();
- });
- }
-
- // Upload file in chunks (max 80 MB per chunk)
+ // Upload file in chunks (max 80 MB per chunk) without computing file hash
async function uploadFileInChunks(file) {
const maxChunkSize = 80 * 1024 * 1024; // 80 MB
let offset = 0;
let uploadId = null;
- try {
- appendLog("Starting hash computation...");
- const hashHex = await computeSHA256(file);
- appendLog(`Computed SHA-256 hash: ${hashHex}`);
+ appendLog("Starting file upload...");
+
+ while (offset < file.size) {
+ const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size));
+ appendLog(`Uploading chunk starting at byte ${offset}`);
- while (offset < file.size) {
- const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size));
- appendLog(`Uploading chunk starting at byte ${offset}`);
-
- // Prepare headers for the chunk upload
- const headers = {
- "X-Content-SHA256": hashHex,
- "X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64
- "X-Chunk-Start": offset.toString(),
- "Content-Type": file.type || "application/octet-stream"
- };
- if (uploadId) {
- headers["X-Upload-ID"] = uploadId;
- }
-
- const response = await fetch(BASE_URL, {
- method: "POST",
- headers: headers,
- body: chunk
- });
-
- if (!response.ok) {
- const errorData = await response.json();
- appendLog(`Chunk upload failed: ${errorData.error}`);
- throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`);
- }
-
- const resultData = await response.json();
- // Save uploadId from first response if not set
- if (!uploadId && resultData.upload_id) {
- uploadId = resultData.upload_id;
- }
-
- // If final response contains content_id, upload is complete
- if (resultData.content_id) {
- appendLog(`Upload complete. File ID: ${resultData.content_id}`);
- return resultData;
- }
-
- // Update offset based on server-reported current size
- if (resultData.current_size !== undefined) {
- offset = resultData.current_size;
- appendLog(`Server reports current_size: ${offset}`);
- } else {
- appendLog("Unexpected response from server, missing current_size.");
- throw new Error("Missing current_size in response");
- }
+ // Prepare headers for the chunk upload
+ const headers = {
+ "X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64
+ "X-Chunk-Start": offset.toString(),
+ "Content-Type": file.type || "application/octet-stream"
+ };
+ if (uploadId) {
+ headers["X-Upload-ID"] = uploadId;
+ }
+ // Set header to indicate the last chunk if this is the final part of the file
+ if (offset + chunk.size >= file.size) {
+ headers["X-Last-Chunk"] = "1";
+ }
+
+ const response = await fetch(BASE_URL, {
+ method: "POST",
+ headers: headers,
+ body: chunk
+ });
+
+ if (!response.ok) {
+ const errorData = await response.json();
+ appendLog(`Chunk upload failed: ${errorData.error}`);
+ throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`);
+ }
+
+ const resultData = await response.json();
+ // Save uploadId from first response if not set
+ if (!uploadId && resultData.upload_id) {
+ uploadId = resultData.upload_id;
+ }
+
+ // If final response contains content_id, upload is complete
+ if (resultData.content_id) {
+ appendLog(`Upload complete. File ID: ${resultData.content_id}`);
+ return resultData;
+ }
+
+ // Update offset based on server-reported current size
+ if (resultData.current_size !== undefined) {
+ offset = resultData.current_size;
+ appendLog(`Server reports current_size: ${offset}`);
+ } else {
+ appendLog("Unexpected response from server, missing current_size.");
+ throw new Error("Missing current_size in response");
}
- } catch (err) {
- appendLog(`Error during upload: ${err}`);
- throw err;
}
}