my-dev
/
uploader-bot
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes

This commit is contained in:
user 2025-02-28 17:30:39 +03:00
parent c8456f0fac
commit bb2180f44b
4 changed files with 74 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/api/middleware.py
View File

@ -15,7 +15,7 @@ from datetime import datetime, timedelta
def attach_headers(response): def attach_headers(response):
response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS" response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-content-sha256, x-chunk-start, x-upload-id" response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
response.headers["Access-Control-Allow-Credentials"] = "true" response.headers["Access-Control-Allow-Credentials"] = "true"
return response return response

2
app/api/routes/content.py
View File

@ -147,8 +147,6 @@ async def s_api_v1_content_view(request, content_address: str):
async def s_api_v1_content_friendly_list(request): async def s_api_v1_content_friendly_list(request):
# return html table with content list. bootstrap is used # return html table with content list. bootstrap is used
result = """ result = """
<html> <html>
<head> <head>

32
app/api/routes/progressive_storage.py
View File

@ -21,33 +21,6 @@ async def s_api_v1_5_storage_post(request):
# Log the receipt of a chunk upload request # Log the receipt of a chunk upload request
make_log("uploader_v1.5", "Received chunk upload request", level="INFO") make_log("uploader_v1.5", "Received chunk upload request", level="INFO")
# Get the provided file hash from header (hex format)
provided_hash_hex = request.headers.get("X-Content-SHA256")
if not provided_hash_hex:
make_log("uploader_v1.5", "Missing X-Content-SHA256 header", level="ERROR")
return response.json({"error": "Missing X-Content-SHA256 header"}, status=400)
try:
provided_hash_bytes = bytes.fromhex(provided_hash_hex)
provided_hash_b58 = b58encode(provided_hash_bytes).decode()
make_log("uploader_v1.5", f"Provided hash (base58): {provided_hash_b58}", level="INFO")
except Exception as e:
make_log("uploader_v1.5", f"Invalid X-Content-SHA256 header format: {e}", level="ERROR")
return response.json({"error": "Invalid X-Content-SHA256 header format"}, status=400)
existing = request.ctx.db_session.query(StoredContent).filter_by(hash=provided_hash_b58).first()
if existing:
make_log("uploader_v1.5", f"File with hash {provided_hash_b58} already exists in DB", level="INFO")
serialized_v2 = existing.cid.serialize_v2() # Get v2 content id
serialized_v1 = existing.cid.serialize_v1() # Get v1 content id
# Return early response since the file is already stored
return response.json({
"upload_id": request.headers.get("X-Upload-ID", str(uuid4())), # Use provided or generate a new upload_id
"content_sha256": provided_hash_b58,
"content_id": serialized_v2,
"content_id_v1": serialized_v1,
"content_url": f"dmy://storage?cid={serialized_v2}",
})
# Get the provided file name from header and decode it from base64 # Get the provided file name from header and decode it from base64
provided_filename_b64 = request.headers.get("X-File-Name") provided_filename_b64 = request.headers.get("X-File-Name")
if not provided_filename_b64: if not provided_filename_b64:
@ -125,6 +98,9 @@ async def s_api_v1_5_storage_post(request):
make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR") make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR")
return response.json({"error": "Failed to save chunk"}, status=500) return response.json({"error": "Failed to save chunk"}, status=500)
# If computed hash matches the provided one, the final chunk has been received
is_last_chunk = int(request.headers.get("X-Last-Chunk", "0")) == 1
if is_last_chunk:
# Compute the SHA256 hash of the temporary file using subprocess # Compute the SHA256 hash of the temporary file using subprocess
try: try:
proc = await asyncio.create_subprocess_exec( proc = await asyncio.create_subprocess_exec(
@ -145,8 +121,6 @@ async def s_api_v1_5_storage_post(request):
make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR") make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
return response.json({"error": "Error computing file hash"}, status=500) return response.json({"error": "Error computing file hash"}, status=500)
# If computed hash matches the provided one, the final chunk has been received
if computed_hash_b58 == provided_hash_b58:
final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}") final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}")
try: try:
os.rename(temp_path, final_path) os.rename(temp_path, final_path)

56
uploader_test.html
View File

@ -10,8 +10,6 @@
input, button { margin-bottom: 10px; } input, button { margin-bottom: 10px; }
#log { border: 1px solid #ccc; padding: 10px; max-height: 200px; overflow-y: auto; background: #f9f9f9; } #log { border: 1px solid #ccc; padding: 10px; max-height: 200px; overflow-y: auto; background: #f9f9f9; }
</style> </style>
<!-- Including jsSHA library from CDN -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/jsSHA/3.2.0/sha256.js"></script>
</head> </head>
<body> <body>
<h1>Загрузка и стриминг файла</h1> <h1>Загрузка и стриминг файла</h1>
@ -51,53 +49,12 @@
logDiv.appendChild(p); logDiv.appendChild(p);
} }
// Compute SHA-256 hash of a file using jsSHA library (incremental reading) // Upload file in chunks (max 80 MB per chunk) without computing file hash
function computeSHA256(file) {
return new Promise((resolve, reject) => {
const chunkSize = 2097152 * 10; // 2MB per chunk
let offset = 0;
const reader = new FileReader();
const shaObj = new jsSHA("SHA-256", "ARRAYBUFFER");
reader.onload = function(e) {
// Update hash with current chunk data
shaObj.update(e.target.result);
offset += chunkSize;
appendLog(`Processed ${Math.min(offset, file.size)} из ${file.size} байт`);
if (offset < file.size) {
readNextChunk();
} else {
try {
const hash = shaObj.getHash("HEX");
resolve(hash);
} catch (err) {
reject(err);
}
}
};
reader.onerror = function(err) {
reject(err);
};
function readNextChunk() {
const slice = file.slice(offset, offset + chunkSize);
reader.readAsArrayBuffer(slice);
}
readNextChunk();
});
}
// Upload file in chunks (max 80 MB per chunk)
async function uploadFileInChunks(file) { async function uploadFileInChunks(file) {
const maxChunkSize = 80 * 1024 * 1024; // 80 MB const maxChunkSize = 80 * 1024 * 1024; // 80 MB
let offset = 0; let offset = 0;
let uploadId = null; let uploadId = null;
try { appendLog("Starting file upload...");
appendLog("Starting hash computation...");
const hashHex = await computeSHA256(file);
appendLog(`Computed SHA-256 hash: ${hashHex}`);
while (offset < file.size) { while (offset < file.size) {
const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size)); const chunk = file.slice(offset, Math.min(offset + maxChunkSize, file.size));
@ -105,7 +62,6 @@
// Prepare headers for the chunk upload // Prepare headers for the chunk upload
const headers = { const headers = {
"X-Content-SHA256": hashHex,
"X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64 "X-File-Name": btoa(unescape(encodeURIComponent(file.name))), // File name in base64
"X-Chunk-Start": offset.toString(), "X-Chunk-Start": offset.toString(),
"Content-Type": file.type || "application/octet-stream" "Content-Type": file.type || "application/octet-stream"
@ -113,6 +69,10 @@
if (uploadId) { if (uploadId) {
headers["X-Upload-ID"] = uploadId; headers["X-Upload-ID"] = uploadId;
} }
// Set header to indicate the last chunk if this is the final part of the file
if (offset + chunk.size >= file.size) {
headers["X-Last-Chunk"] = "1";
}
const response = await fetch(BASE_URL, { const response = await fetch(BASE_URL, {
method: "POST", method: "POST",
@ -147,10 +107,6 @@
throw new Error("Missing current_size in response"); throw new Error("Missing current_size in response");
} }
} }
} catch (err) {
appendLog(`Error during upload: ${err}`);
throw err;
}
} }
// Upload button event listener // Upload button event listener