my-dev
/
uploader-bot
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes

This commit is contained in:
user 2025-02-28 17:30:39 +03:00
parent c8456f0fac
commit bb2180f44b
4 changed files with 74 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/api/middleware.py
View File

@ -15,7 +15,7 @@ from datetime import datetime, timedelta
def attach_headers(response): def attach_headers(response):
response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS" response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-content-sha256, x-chunk-start, x-upload-id" response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
response.headers["Access-Control-Allow-Credentials"] = "true" response.headers["Access-Control-Allow-Credentials"] = "true"
return response return response

2
app/api/routes/content.py
View File

@ -147,8 +147,6 @@ async def s_api_v1_content_view(request, content_address: str):
async def s_api_v1_content_friendly_list(request): async def s_api_v1_content_friendly_list(request):
# return html table with content list. bootstrap is used # return html table with content list. bootstrap is used
result = """ result = """
<html> <html>
<head> <head>

70
app/api/routes/progressive_storage.py
View File

@ -21,33 +21,6 @@ async def s_api_v1_5_storage_post(request):
# Log the receipt of a chunk upload request # Log the receipt of a chunk upload request
make_log("uploader_v1.5", "Received chunk upload request", level="INFO") make_log("uploader_v1.5", "Received chunk upload request", level="INFO")
# Get the provided file hash from header (hex format)
provided_hash_hex = request.headers.get("X-Content-SHA256")
if not provided_hash_hex:
make_log("uploader_v1.5", "Missing X-Content-SHA256 header", level="ERROR")
return response.json({"error": "Missing X-Content-SHA256 header"}, status=400)
try:
provided_hash_bytes = bytes.fromhex(provided_hash_hex)
provided_hash_b58 = b58encode(provided_hash_bytes).decode()
make_log("uploader_v1.5", f"Provided hash (base58): {provided_hash_b58}", level="INFO")
except Exception as e:
make_log("uploader_v1.5", f"Invalid X-Content-SHA256 header format: {e}", level="ERROR")
return response.json({"error": "Invalid X-Content-SHA256 header format"}, status=400)
existing = request.ctx.db_session.query(StoredContent).filter_by(hash=provided_hash_b58).first()
if existing:
make_log("uploader_v1.5", f"File with hash {provided_hash_b58} already exists in DB", level="INFO")
serialized_v2 = existing.cid.serialize_v2() # Get v2 content id
serialized_v1 = existing.cid.serialize_v1() # Get v1 content id
# Return early response since the file is already stored
return response.json({
"upload_id": request.headers.get("X-Upload-ID", str(uuid4())), # Use provided or generate a new upload_id
"content_sha256": provided_hash_b58,
"content_id": serialized_v2,
"content_id_v1": serialized_v1,
"content_url": f"dmy://storage?cid={serialized_v2}",
})
# Get the provided file name from header and decode it from base64 # Get the provided file name from header and decode it from base64
provided_filename_b64 = request.headers.get("X-File-Name") provided_filename_b64 = request.headers.get("X-File-Name")
if not provided_filename_b64: if not provided_filename_b64:
@ -125,28 +98,29 @@ async def s_api_v1_5_storage_post(request):
make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR") make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR")
return response.json({"error": "Failed to save chunk"}, status=500) return response.json({"error": "Failed to save chunk"}, status=500)
# Compute the SHA256 hash of the temporary file using subprocess
try:
proc = await asyncio.create_subprocess_exec(
'sha256sum', temp_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
error_msg = stderr.decode().strip()
make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
return response.json({"error": "Failed to compute file hash"}, status=500)
computed_hash_hex = stdout.decode().split()[0].strip()
computed_hash_bytes = bytes.fromhex(computed_hash_hex)
computed_hash_b58 = b58encode(computed_hash_bytes).decode()
make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
except Exception as e:
make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
return response.json({"error": "Error computing file hash"}, status=500)
# If computed hash matches the provided one, the final chunk has been received # If computed hash matches the provided one, the final chunk has been received
if computed_hash_b58 == provided_hash_b58: is_last_chunk = int(request.headers.get("X-Last-Chunk", "0")) == 1
if is_last_chunk:
# Compute the SHA256 hash of the temporary file using subprocess
try:
proc = await asyncio.create_subprocess_exec(
'sha256sum', temp_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
error_msg = stderr.decode().strip()
make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
return response.json({"error": "Failed to compute file hash"}, status=500)
computed_hash_hex = stdout.decode().split()[0].strip()
computed_hash_bytes = bytes.fromhex(computed_hash_hex)
computed_hash_b58 = b58encode(computed_hash_bytes).decode()
make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
except Exception as e:
make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
return response.json({"error": "Error computing file hash"}, status=500)
final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}") final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}")
try: try:
os.rename(temp_path, final_path) os.rename(temp_path, final_path)

144
uploader_test.html
View File

@ -10,8 +10,6 @@
input, button { margin-bottom: 10px; } input, button { margin-bottom: 10px; }
#log { border: 1px solid #ccc; padding: 10px; max-height: 200px; overflow-y: auto; background: #f9f9f9; } #log { border: 1px solid #ccc; padding: 10px; max-height: 200px; overflow-y: auto; background: #f9f9f9; }
</style> </style>
<!-- Including jsSHA library from CDN -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/jsSHA/3.2.0/sha256.js"></script>
</head> </head>
<body> <body>
<h1>Загрузка и стриминг файла</h1> <h1>Загрузка и стриминг файла</h1>
@ -51,105 +49,63 @@
logDiv.appendChild(p); logDiv.appendChild(p);
} }
// Compute SHA-256 hash of a file using jsSHA library (incremental reading) // Upload file in chunks (max 80 MB per chunk) without computing file hash
function computeSHA256(file) {
return new Promise((resolve, reject) => {
const chunkSize = 2097152 * 10; // 2MB per chunk
let offset = 0;
const reader = new FileReader();
const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");
reader.onload = function(e) {
// Update hash with current chunk data
shaObj.update(e.target.result);
offset += chunkSize;
appendLog(`Processed ${Math.min(offset, file.size)} из ${file.size} байт`);
if (offset < file.size) {
readNextChunk();
} else {
try {
const hash = shaObj.getHash("HEX");
resolve(hash);
} catch (err) {
reject(err);
}
}
};
reader.onerror = function(err) {
reject(err);
};
function readNextChunk() {
const slice = file.slice(offset, offset + chunkSize);
reader.readAsArrayBuffer(slice);
}
readNextChunk();
});
}
// Upload file in chunks (max 80 MB per chunk)
async function uploadFileInChunks(file) { async function uploadFileInChunks(file) {
const maxChunkSize = 80 * 1024 * 1024; // 80 MB const maxChunkSize = 80 * 1024 * 1024; // 80 MB
let offset = 0; let offset = 0;
let uploadId = null; let uploadId = null;
try { appendLog("Starting file upload...");
appendLog("Starting hash computation...");
const hashHex = await computeSHA256(file);
appendLog(`Computed SHA-256 hash: ${hashHex}`);
while (offset < file.size) { while (offset < file.size) {
const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size)); const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size));
appendLog(`Uploading chunk starting at byte ${offset}`); appendLog(`Uploading chunk starting at byte ${offset}`);
// Prepare headers for the chunk upload // Prepare headers for the chunk upload
const headers = { const headers = {
"X-Content-SHA256": hashHex, "X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64
"X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64 "X-Chunk-Start": offset.toString(),
"X-Chunk-Start": offset.toString(), "Content-Type": file.type || "application/octet-stream"
"Content-Type": file.type || "application/octet-stream" };
}; if (uploadId) {
if (uploadId) { headers["X-Upload-ID"] = uploadId;
headers["X-Upload-ID"] = uploadId; }
} // Set header to indicate the last chunk if this is the final part of the file
if (offset + chunk.size >= file.size) {
const response = await fetch(BASE_URL, { headers["X-Last-Chunk"] = "1";
method: "POST", }
headers: headers,
body: chunk const response = await fetch(BASE_URL, {
}); method: "POST",
headers: headers,
if (!response.ok) { body: chunk
const errorData = await response.json(); });
appendLog(`Chunk upload failed: ${errorData.error}`);
throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`); if (!response.ok) {
} const errorData = await response.json();
appendLog(`Chunk upload failed: ${errorData.error}`);
const resultData = await response.json(); throw new Error(`Upload failed at offset ${offset}: ${errorData.error}`);
// Save uploadId from first response if not set }
if (!uploadId && resultData.upload_id) {
uploadId = resultData.upload_id; const resultData = await response.json();
} // Save uploadId from first response if not set
if (!uploadId && resultData.upload_id) {
// If final response contains content_id, upload is complete uploadId = resultData.upload_id;
if (resultData.content_id) { }
appendLog(`Upload complete. File ID: ${resultData.content_id}`);
return resultData; // If final response contains content_id, upload is complete
} if (resultData.content_id) {
appendLog(`Upload complete. File ID: ${resultData.content_id}`);
// Update offset based on server-reported current size return resultData;
if (resultData.current_size !== undefined) { }
offset = resultData.current_size;
appendLog(`Server reports current_size: ${offset}`); // Update offset based on server-reported current size
} else { if (resultData.current_size !== undefined) {
appendLog("Unexpected response from server, missing current_size."); offset = resultData.current_size;
throw new Error("Missing current_size in response"); appendLog(`Server reports current_size: ${offset}`);
} } else {
appendLog("Unexpected response from server, missing current_size.");
throw new Error("Missing current_size in response");
} }
} catch (err) {
appendLog(`Error during upload: ${err}`);
throw err;
} }
} }