my-dev
/
uploader-bot
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

dev@locazia: migrate to cidv2

This commit is contained in:
user 2024-03-08 19:27:51 +03:00
parent 3fdc9ada39
commit 606ef553bc
4 changed files with 98 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/api/routes/_blockchain.py
View File

@ -119,14 +119,14 @@ async def s_api_v1_blockchain_send_new_content_message(request):
.store_ref(
begin_cell()
.store_uint(1, 8)
.store_bytes(f"{PROJECT_HOST}/api/v1/storage/{metadata_content.cid.serialize_v1()}".encode())
.store_bytes(f"{PROJECT_HOST}/api/v1/storage/{metadata_content.cid.serialize_v2()}".encode())
.end_cell()
)
.store_ref(
begin_cell()
.store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v1()}".encode()).end_cell())
.store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v1() if image_content_cid else ''}".encode()).end_cell())
.store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v1()}".encode()).end_cell())
.store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v2()}".encode()).end_cell())
.store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v2() if image_content_cid else ''}".encode()).end_cell())
.store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v2()}".encode()).end_cell())
.end_cell()
)
.end_cell()

16
app/api/routes/node_storage.py
View File

@ -41,10 +41,12 @@ async def s_api_v1_storage_post(request):
stored_content = request.ctx.db_session.query(StoredContent).filter(StoredContent.hash == file_hash).first()
if stored_content:
stored_cid = stored_content.cid.serialize_v1()
stored_cid_v2 = stored_content.cid.serialize_v2()
return response.json({
"content_sha256": file_hash,
"content_id_v1": stored_cid,
"content_url": f"dmy://storage?cid={stored_cid}",
"content_id": stored_cid_v2,
"content_url": f"dmy://storage?cid={stored_cid_v2}"
})
if request.ctx.user:
@ -71,11 +73,13 @@ async def s_api_v1_storage_post(request):
await file.write(file_content)
new_content_id = new_content.cid
new_cid = new_content_id.serialize_v1()
new_cid_v1 = new_content.serialize_v1()
new_cid = new_content_id.serialize_v2()
return response.json({
"content_sha256": file_hash,
"content_id_v1": new_cid,
"content_id": new_cid,
"content_id_v1": new_cid_v1,
"content_url": f"dmy://storage?cid={new_cid}",
})
except BaseException as e:
@ -107,8 +111,4 @@ async def s_api_v1_storage_decode_cid(request, content_id=None):
if errmsg:
return response.json({"error": errmsg}, status=400)
return response.json({
"content_hash": b58encode(cid.content_hash).decode(),
"onchain_index": cid.onchain_index,
"accept_type": cid.accept_type,
})
return response.json(cid.json_format())

88
app/core/content/content_id.py
View File

@ -1,28 +1,65 @@
from base58 import b58encode, b58decode
import math
from tonsdk.boc import begin_cell
from app.core._config import ALLOWED_CONTENT_TYPES
from app.core._utils.string_binary import string_to_bytes_fixed_size, bytes_to_string
# cid_v1#_ cid_version:int8 accept_type:uint120 content_sha256:uint256 onchain_index:uint128 = CIDv1;
# cid_v1#_ cid_version:uint8 accept_type:uint120 content_sha256:uint256 onchain_index:uint128 = CIDv1;
# onchain_index#b0 bytes_len:uint8 index:uint_var = OnchainIndex;
# accept_type#b1 bytes_len:uint8 value:bytes = Param;
# encryption_key_sha256#b2 digest:uint256 = EncryptionKey;
# cid_v2#_ cid_version:uint8 content_sha256:uint256 *[Param]s = CIDv2;
class ContentId:
def __init__(
self,
version: int = None,
content_hash: bytes = None, # only SHA256
onchain_index: int = None,
accept_type: str = 'image/jpeg'
accept_type: str = 'image/jpeg',
encryption_key_sha256: bytes = None,
):
self.version = version
self.content_hash = content_hash
self.onchain_index = onchain_index or -1
self.onchain_index = onchain_index or -1
self.accept_type = accept_type
self.encryption_key_sha256 = encryption_key_sha256
if self.encryption_key_sha256:
assert len(self.encryption_key_sha256) == 32, "Invalid encryption key length"
@property
def content_hash_b58(self) -> str:
return b58encode(self.content_hash).decode()
@property
def safe_onchain_index(self):
return self.onchain_index if (not (self.onchain_index is None) and self.onchain_index >= 0) else None
def serialize_v2(self) -> str:
cid_bin = (
(2).to_bytes(1, 'big') # cid version
+ self.content_hash
)
if not (self.safe_onchain_index is None):
oi_bin_hex = hex(self.safe_onchain_index)[2:]
oi_bin_len = math.ceil(len(oi_bin_hex) / 2)
cid_bin += b'\xb0' + oi_bin_len.to_bytes(1, 'big') + bytes.fromhex(oi_bin_hex)
if self.accept_type:
at_bin_len = len(self.accept_type.encode())
at_bin = string_to_bytes_fixed_size(self.accept_type, at_bin_len)
cid_bin += (
b'\xb1'
+ at_bin_len.to_bytes(1, 'big')
+ at_bin
)
if self.encryption_key_sha256:
cid_bin += b'\xb2' + self.encryption_key_sha256
return b58encode(cid_bin).decode()
def serialize_v1(self) -> str:
at_bin = string_to_bytes_fixed_size(self.accept_type, 15)
assert len(self.content_hash) == 32, "Invalid hash length"
@ -39,6 +76,37 @@ class ContentId:
+ oi_bin
).decode()
@classmethod
def from_v2(cls, cid: str):
cid_bin = b58decode(cid)
(
cid_version,
content_sha256,
cid_bin
) = (
int.from_bytes(cid_bin[0:1], 'big'),
cid_bin[1:33],
cid_bin[33:]
)
assert cid_version == 2, "Invalid version"
params = {}
while cid_bin:
param_op = cid_bin[0:1]
cid_bin = cid_bin[1:]
if param_op == b'\xb0': # onchain_index
oi_len = int.from_bytes(cid_bin[0:1], 'big')
params['onchain_index'] = int.from_bytes(cid_bin[1:1 + oi_len], 'big')
cid_bin = cid_bin[1 + oi_len:]
elif param_op == b'\xb1': # accept_type
at_len = int.from_bytes(cid_bin[0:1], 'big')
params['accept_type'] = bytes_to_string(cid_bin[1:1 + at_len])
cid_bin = cid_bin[1 + at_len:]
elif param_op == b'\xb2': # encryption_key_sha256
params['encryption_key_sha256'] = cid_bin[1:33]
cid_bin = cid_bin[33:]
return cls(version=2, content_hash=content_sha256, **params)
@classmethod
def from_v1(cls, cid: str):
cid_bin = b58decode(cid)
@ -58,6 +126,7 @@ class ContentId:
# assert '/'.join(content_type[0:2]) in ALLOWED_CONTENT_TYPES, "Invalid accept type"
assert len(content_sha256) == 32, "Invalid hash length"
return cls(
version=1,
content_hash=content_sha256,
onchain_index=onchain_index,
accept_type=accept_type
@ -68,7 +137,18 @@ class ContentId:
cid_version = int.from_bytes(b58decode(cid)[0:1], 'big')
if cid_version == 1:
return cls.from_v1(cid)
elif cid_version == 2:
return cls.from_v2(cid)
else:
raise ValueError("Invalid cid version")
def json_format(self):
return {
"version": self.version,
"content_hash": self.content_hash_b58,
"onchain_index": self.safe_onchain_index,
"accept_type": self.accept_type,
"encryption_key_sha256": b58encode(self.encryption_key_sha256).decode() if self.encryption_key_sha256 else None
}

3
app/core/models/node_storage.py
View File

@ -14,6 +14,7 @@ class StoredContent(AlchemyBase):
id = Column(Integer, autoincrement=True, primary_key=True)
type = Column(String(32), nullable=False)
hash = Column(String(64), nullable=False, unique=True) # base58
content_id = Column(String(512), nullable=True) # base58
onchain_index = Column(BigInteger, nullable=True, default=None)
status = Column(String(32), nullable=True)
@ -70,7 +71,7 @@ class StoredContent(AlchemyBase):
return {
**extra_fields,
"hash": self.hash,
"cid": self.cid.serialize_v1(),
"cid": self.cid.serialize_v2(),
"status": self.status,
"updated": self.updated.isoformat() if isinstance(self.updated, datetime) else (make_log("Content.json_format", f"Invalid Content.updated: {self.updated} ({type(self.updated)})", level="error") or None),
"created": self.created.isoformat() if isinstance(self.created, datetime) else (make_log("Content.json_format", f"Invalid Content.created: {self.created} ({type(self.created)})", level="error") or None),