diff --git a/app/api/routes/_blockchain.py b/app/api/routes/_blockchain.py index d482326..0dfbd16 100644 --- a/app/api/routes/_blockchain.py +++ b/app/api/routes/_blockchain.py @@ -119,14 +119,14 @@ async def s_api_v1_blockchain_send_new_content_message(request): .store_ref( begin_cell() .store_uint(1, 8) - .store_bytes(f"{PROJECT_HOST}/api/v1/storage/{metadata_content.cid.serialize_v1()}".encode()) + .store_bytes(f"{PROJECT_HOST}/api/v1/storage/{metadata_content.cid.serialize_v2()}".encode()) .end_cell() ) .store_ref( begin_cell() - .store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v1()}".encode()).end_cell()) - .store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v1() if image_content_cid else ''}".encode()).end_cell()) - .store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v1()}".encode()).end_cell()) + .store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v2()}".encode()).end_cell()) + .store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v2() if image_content_cid else ''}".encode()).end_cell()) + .store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v2()}".encode()).end_cell()) .end_cell() ) .end_cell() diff --git a/app/api/routes/node_storage.py b/app/api/routes/node_storage.py index 0547439..73063cc 100644 --- a/app/api/routes/node_storage.py +++ b/app/api/routes/node_storage.py @@ -41,10 +41,12 @@ async def s_api_v1_storage_post(request): stored_content = request.ctx.db_session.query(StoredContent).filter(StoredContent.hash == file_hash).first() if stored_content: stored_cid = stored_content.cid.serialize_v1() + stored_cid_v2 = stored_content.cid.serialize_v2() return response.json({ "content_sha256": file_hash, "content_id_v1": stored_cid, - "content_url": f"dmy://storage?cid={stored_cid}", + "content_id": stored_cid_v2, + "content_url": f"dmy://storage?cid={stored_cid_v2}" }) if request.ctx.user: @@ -71,11 +73,13 @@ async def s_api_v1_storage_post(request): await file.write(file_content) new_content_id = new_content.cid - new_cid = new_content_id.serialize_v1() + new_cid_v1 = new_content.serialize_v1() + new_cid = new_content_id.serialize_v2() return response.json({ "content_sha256": file_hash, - "content_id_v1": new_cid, + "content_id": new_cid, + "content_id_v1": new_cid_v1, "content_url": f"dmy://storage?cid={new_cid}", }) except BaseException as e: @@ -107,8 +111,4 @@ async def s_api_v1_storage_decode_cid(request, content_id=None): if errmsg: return response.json({"error": errmsg}, status=400) - return response.json({ - "content_hash": b58encode(cid.content_hash).decode(), - "onchain_index": cid.onchain_index, - "accept_type": cid.accept_type, - }) + return response.json(cid.json_format()) diff --git a/app/core/content/content_id.py b/app/core/content/content_id.py index b976b9f..fafb698 100644 --- a/app/core/content/content_id.py +++ b/app/core/content/content_id.py @@ -1,28 +1,65 @@ from base58 import b58encode, b58decode +import math +from tonsdk.boc import begin_cell from app.core._config import ALLOWED_CONTENT_TYPES from app.core._utils.string_binary import string_to_bytes_fixed_size, bytes_to_string -# cid_v1#_ cid_version:int8 accept_type:uint120 content_sha256:uint256 onchain_index:uint128 = CIDv1; - +# cid_v1#_ cid_version:uint8 accept_type:uint120 content_sha256:uint256 onchain_index:uint128 = CIDv1; +# onchain_index#b0 bytes_len:uint8 index:uint_var = OnchainIndex; +# accept_type#b1 bytes_len:uint8 value:bytes = Param; +# encryption_key_sha256#b2 digest:uint256 = EncryptionKey; +# cid_v2#_ cid_version:uint8 content_sha256:uint256 *[Param]s = CIDv2; class ContentId: def __init__( self, + version: int = None, content_hash: bytes = None, # only SHA256 onchain_index: int = None, - accept_type: str = 'image/jpeg' + accept_type: str = 'image/jpeg', + encryption_key_sha256: bytes = None, ): + self.version = version self.content_hash = content_hash - self.onchain_index = onchain_index or -1 + self.onchain_index = onchain_index or -1 self.accept_type = accept_type + self.encryption_key_sha256 = encryption_key_sha256 + if self.encryption_key_sha256: + assert len(self.encryption_key_sha256) == 32, "Invalid encryption key length" @property def content_hash_b58(self) -> str: return b58encode(self.content_hash).decode() + @property + def safe_onchain_index(self): + return self.onchain_index if (not (self.onchain_index is None) and self.onchain_index >= 0) else None + + def serialize_v2(self) -> str: + cid_bin = ( + (2).to_bytes(1, 'big') # cid version + + self.content_hash + ) + if not (self.safe_onchain_index is None): + oi_bin_hex = hex(self.safe_onchain_index)[2:] + oi_bin_len = math.ceil(len(oi_bin_hex) / 2) + cid_bin += b'\xb0' + oi_bin_len.to_bytes(1, 'big') + bytes.fromhex(oi_bin_hex) + if self.accept_type: + at_bin_len = len(self.accept_type.encode()) + at_bin = string_to_bytes_fixed_size(self.accept_type, at_bin_len) + cid_bin += ( + b'\xb1' + + at_bin_len.to_bytes(1, 'big') + + at_bin + ) + if self.encryption_key_sha256: + cid_bin += b'\xb2' + self.encryption_key_sha256 + + return b58encode(cid_bin).decode() + def serialize_v1(self) -> str: at_bin = string_to_bytes_fixed_size(self.accept_type, 15) assert len(self.content_hash) == 32, "Invalid hash length" @@ -39,6 +76,37 @@ class ContentId: + oi_bin ).decode() + @classmethod + def from_v2(cls, cid: str): + cid_bin = b58decode(cid) + ( + cid_version, + content_sha256, + cid_bin + ) = ( + int.from_bytes(cid_bin[0:1], 'big'), + cid_bin[1:33], + cid_bin[33:] + ) + assert cid_version == 2, "Invalid version" + params = {} + while cid_bin: + param_op = cid_bin[0:1] + cid_bin = cid_bin[1:] + if param_op == b'\xb0': # onchain_index + oi_len = int.from_bytes(cid_bin[0:1], 'big') + params['onchain_index'] = int.from_bytes(cid_bin[1:1 + oi_len], 'big') + cid_bin = cid_bin[1 + oi_len:] + elif param_op == b'\xb1': # accept_type + at_len = int.from_bytes(cid_bin[0:1], 'big') + params['accept_type'] = bytes_to_string(cid_bin[1:1 + at_len]) + cid_bin = cid_bin[1 + at_len:] + elif param_op == b'\xb2': # encryption_key_sha256 + params['encryption_key_sha256'] = cid_bin[1:33] + cid_bin = cid_bin[33:] + + return cls(version=2, content_hash=content_sha256, **params) + @classmethod def from_v1(cls, cid: str): cid_bin = b58decode(cid) @@ -58,6 +126,7 @@ class ContentId: # assert '/'.join(content_type[0:2]) in ALLOWED_CONTENT_TYPES, "Invalid accept type" assert len(content_sha256) == 32, "Invalid hash length" return cls( + version=1, content_hash=content_sha256, onchain_index=onchain_index, accept_type=accept_type @@ -68,7 +137,18 @@ class ContentId: cid_version = int.from_bytes(b58decode(cid)[0:1], 'big') if cid_version == 1: return cls.from_v1(cid) + elif cid_version == 2: + return cls.from_v2(cid) else: raise ValueError("Invalid cid version") + def json_format(self): + return { + "version": self.version, + "content_hash": self.content_hash_b58, + "onchain_index": self.safe_onchain_index, + "accept_type": self.accept_type, + "encryption_key_sha256": b58encode(self.encryption_key_sha256).decode() if self.encryption_key_sha256 else None + } + diff --git a/app/core/models/node_storage.py b/app/core/models/node_storage.py index b70a2f5..aacc8ad 100644 --- a/app/core/models/node_storage.py +++ b/app/core/models/node_storage.py @@ -14,6 +14,7 @@ class StoredContent(AlchemyBase): id = Column(Integer, autoincrement=True, primary_key=True) type = Column(String(32), nullable=False) hash = Column(String(64), nullable=False, unique=True) # base58 + content_id = Column(String(512), nullable=True) # base58 onchain_index = Column(BigInteger, nullable=True, default=None) status = Column(String(32), nullable=True) @@ -70,7 +71,7 @@ class StoredContent(AlchemyBase): return { **extra_fields, "hash": self.hash, - "cid": self.cid.serialize_v1(), + "cid": self.cid.serialize_v2(), "status": self.status, "updated": self.updated.isoformat() if isinstance(self.updated, datetime) else (make_log("Content.json_format", f"Invalid Content.updated: {self.updated} ({type(self.updated)})", level="error") or None), "created": self.created.isoformat() if isinstance(self.created, datetime) else (make_log("Content.json_format", f"Invalid Content.created: {self.created} ({type(self.created)})", level="error") or None),