fixes
This commit is contained in:
user
parent
c8456f0fac
commit
bb2180f44b
|
|
@ -15,7 +15,7 @@ from datetime import datetime, timedelta
|
|||
def attach_headers(response):
|
||||
response.headers["Access-Control-Allow-Origin"] = "*"
|
||||
response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
|
||||
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-content-sha256, x-chunk-start, x-upload-id"
|
||||
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
|
||||
response.headers["Access-Control-Allow-Credentials"] = "true"
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -147,8 +147,6 @@ async def s_api_v1_content_view(request, content_address: str):
|
|||
async def s_api_v1_content_friendly_list(request):
|
||||
# return html table with content list. bootstrap is used
|
||||
|
||||
|
||||
|
||||
result = """
|
||||
<html>
|
||||
<head>
|
||||
|
|
|
|||
|
|
@ -21,33 +21,6 @@ async def s_api_v1_5_storage_post(request):
|
|||
# Log the receipt of a chunk upload request
|
||||
make_log("uploader_v1.5", "Received chunk upload request", level="INFO")
|
||||
|
||||
# Get the provided file hash from header (hex format)
|
||||
provided_hash_hex = request.headers.get("X-Content-SHA256")
|
||||
if not provided_hash_hex:
|
||||
make_log("uploader_v1.5", "Missing X-Content-SHA256 header", level="ERROR")
|
||||
return response.json({"error": "Missing X-Content-SHA256 header"}, status=400)
|
||||
try:
|
||||
provided_hash_bytes = bytes.fromhex(provided_hash_hex)
|
||||
provided_hash_b58 = b58encode(provided_hash_bytes).decode()
|
||||
make_log("uploader_v1.5", f"Provided hash (base58): {provided_hash_b58}", level="INFO")
|
||||
except Exception as e:
|
||||
make_log("uploader_v1.5", f"Invalid X-Content-SHA256 header format: {e}", level="ERROR")
|
||||
return response.json({"error": "Invalid X-Content-SHA256 header format"}, status=400)
|
||||
|
||||
existing = request.ctx.db_session.query(StoredContent).filter_by(hash=provided_hash_b58).first()
|
||||
if existing:
|
||||
make_log("uploader_v1.5", f"File with hash {provided_hash_b58} already exists in DB", level="INFO")
|
||||
serialized_v2 = existing.cid.serialize_v2() # Get v2 content id
|
||||
serialized_v1 = existing.cid.serialize_v1() # Get v1 content id
|
||||
# Return early response since the file is already stored
|
||||
return response.json({
|
||||
"upload_id": request.headers.get("X-Upload-ID", str(uuid4())), # Use provided or generate a new upload_id
|
||||
"content_sha256": provided_hash_b58,
|
||||
"content_id": serialized_v2,
|
||||
"content_id_v1": serialized_v1,
|
||||
"content_url": f"dmy://storage?cid={serialized_v2}",
|
||||
})
|
||||
|
||||
# Get the provided file name from header and decode it from base64
|
||||
provided_filename_b64 = request.headers.get("X-File-Name")
|
||||
if not provided_filename_b64:
|
||||
|
|
@ -125,28 +98,29 @@ async def s_api_v1_5_storage_post(request):
|
|||
make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR")
|
||||
return response.json({"error": "Failed to save chunk"}, status=500)
|
||||
|
||||
# Compute the SHA256 hash of the temporary file using subprocess
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
'sha256sum', temp_path,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
error_msg = stderr.decode().strip()
|
||||
make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
|
||||
return response.json({"error": "Failed to compute file hash"}, status=500)
|
||||
computed_hash_hex = stdout.decode().split()[0].strip()
|
||||
computed_hash_bytes = bytes.fromhex(computed_hash_hex)
|
||||
computed_hash_b58 = b58encode(computed_hash_bytes).decode()
|
||||
make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
|
||||
except Exception as e:
|
||||
make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
|
||||
return response.json({"error": "Error computing file hash"}, status=500)
|
||||
|
||||
# If computed hash matches the provided one, the final chunk has been received
|
||||
if computed_hash_b58 == provided_hash_b58:
|
||||
is_last_chunk = int(request.headers.get("X-Last-Chunk", "0")) == 1
|
||||
if is_last_chunk:
|
||||
# Compute the SHA256 hash of the temporary file using subprocess
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
'sha256sum', temp_path,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
error_msg = stderr.decode().strip()
|
||||
make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
|
||||
return response.json({"error": "Failed to compute file hash"}, status=500)
|
||||
computed_hash_hex = stdout.decode().split()[0].strip()
|
||||
computed_hash_bytes = bytes.fromhex(computed_hash_hex)
|
||||
computed_hash_b58 = b58encode(computed_hash_bytes).decode()
|
||||
make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
|
||||
except Exception as e:
|
||||
make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
|
||||
return response.json({"error": "Error computing file hash"}, status=500)
|
||||
|
||||
final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}")
|
||||
try:
|
||||
os.rename(temp_path, final_path)
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@
|
|||
input, button { margin-bottom: 10px; }
|
||||
#log { border: 1px solid #ccc; padding: 10px; max-height: 200px; overflow-y: auto; background: #f9f9f9; }
|
||||
</style>
|
||||
<!-- Including jsSHA library from CDN -->
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/jsSHA/3.2.0/sha256.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Загрузка и стриминг файла</h1>
|
||||
|
|
@ -51,105 +49,63 @@
|
|||
logDiv.appendChild(p);
|
||||
}
|
||||
|
||||
// Compute SHA-256 hash of a file using jsSHA library (incremental reading)
|
||||
function computeSHA256(file) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunkSize = 2097152 * 10; // 2MB per chunk
|
||||
let offset = 0;
|
||||
const reader = new FileReader();
|
||||
const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");
|
||||
|
||||
reader.onload = function(e) {
|
||||
// Update hash with current chunk data
|
||||
shaObj.update(e.target.result);
|
||||
offset += chunkSize;
|
||||
appendLog(`Processed ${Math.min(offset, file.size)} из ${file.size} байт`);
|
||||
if (offset < file.size) {
|
||||
readNextChunk();
|
||||
} else {
|
||||
try {
|
||||
const hash = shaObj.getHash("HEX");
|
||||
resolve(hash);
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
reader.onerror = function(err) {
|
||||
reject(err);
|
||||
};
|
||||
|
||||
function readNextChunk() {
|
||||
const slice = file.slice(offset, offset + chunkSize);
|
||||
reader.readAsArrayBuffer(slice);
|
||||
}
|
||||
|
||||
readNextChunk();
|
||||
});
|
||||
}
|
||||
|
||||
// Upload file in chunks (max 80 MB per chunk)
|
||||
// Upload file in chunks (max 80 MB per chunk) without computing file hash
|
||||
async function uploadFileInChunks(file) {
|
||||
const maxChunkSize = 80 * 1024 * 1024; // 80 MB
|
||||
let offset = 0;
|
||||
let uploadId = null;
|
||||
try {
|
||||
appendLog("Starting hash computation...");
|
||||
const hashHex = await computeSHA256(file);
|
||||
appendLog(`Computed SHA-256 hash: ${hashHex}`);
|
||||
appendLog("Starting file upload...");
|
||||
|
||||
while (offset < file.size) {
|
||||
const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size));
|
||||
appendLog(`Uploading chunk starting at byte ${offset}`);
|
||||
|
||||
while (offset < file.size) {
|
||||
const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size));
|
||||
appendLog(`Uploading chunk starting at byte ${offset}`);
|
||||
|
||||
// Prepare headers for the chunk upload
|
||||
const headers = {
|
||||
"X-Content-SHA256": hashHex,
|
||||
"X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64
|
||||
"X-Chunk-Start": offset.toString(),
|
||||
"Content-Type": file.type || "application/octet-stream"
|
||||
};
|
||||
if (uploadId) {
|
||||
headers["X-Upload-ID"] = uploadId;
|
||||
}
|
||||
|
||||
const response = await fetch(BASE_URL, {
|
||||
method: "POST",
|
||||
headers: headers,
|
||||
body: chunk
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
appendLog(`Chunk upload failed: ${errorData.error}`);
|
||||
throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`);
|
||||
}
|
||||
|
||||
const resultData = await response.json();
|
||||
// Save uploadId from first response if not set
|
||||
if (!uploadId && resultData.upload_id) {
|
||||
uploadId = resultData.upload_id;
|
||||
}
|
||||
|
||||
// If final response contains content_id, upload is complete
|
||||
if (resultData.content_id) {
|
||||
appendLog(`Upload complete. File ID: ${resultData.content_id}`);
|
||||
return resultData;
|
||||
}
|
||||
|
||||
// Update offset based on server-reported current size
|
||||
if (resultData.current_size !== undefined) {
|
||||
offset = resultData.current_size;
|
||||
appendLog(`Server reports current_size: ${offset}`);
|
||||
} else {
|
||||
appendLog("Unexpected response from server, missing current_size.");
|
||||
throw new Error("Missing current_size in response");
|
||||
}
|
||||
// Prepare headers for the chunk upload
|
||||
const headers = {
|
||||
"X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64
|
||||
"X-Chunk-Start": offset.toString(),
|
||||
"Content-Type": file.type || "application/octet-stream"
|
||||
};
|
||||
if (uploadId) {
|
||||
headers["X-Upload-ID"] = uploadId;
|
||||
}
|
||||
// Set header to indicate the last chunk if this is the final part of the file
|
||||
if (offset + chunk.size >= file.size) {
|
||||
headers["X-Last-Chunk"] = "1";
|
||||
}
|
||||
|
||||
const response = await fetch(BASE_URL, {
|
||||
method: "POST",
|
||||
headers: headers,
|
||||
body: chunk
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
appendLog(`Chunk upload failed: ${errorData.error}`);
|
||||
throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`);
|
||||
}
|
||||
|
||||
const resultData = await response.json();
|
||||
// Save uploadId from first response if not set
|
||||
if (!uploadId && resultData.upload_id) {
|
||||
uploadId = resultData.upload_id;
|
||||
}
|
||||
|
||||
// If final response contains content_id, upload is complete
|
||||
if (resultData.content_id) {
|
||||
appendLog(`Upload complete. File ID: ${resultData.content_id}`);
|
||||
return resultData;
|
||||
}
|
||||
|
||||
// Update offset based on server-reported current size
|
||||
if (resultData.current_size !== undefined) {
|
||||
offset = resultData.current_size;
|
||||
appendLog(`Server reports current_size: ${offset}`);
|
||||
} else {
|
||||
appendLog("Unexpected response from server, missing current_size.");
|
||||
throw new Error("Missing current_size in response");
|
||||
}
|
||||
} catch (err) {
|
||||
appendLog(`Error during upload: ${err}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue