dev@locazia: add service healthy montoring

app/api/routes/_system.py

@@ -1,10 +1,11 @@
 from sanic import response
-from base58 import b58encode
+from base58 import b58encode, b58decode
-from app.core._secrets import hot_pubkey, service_wallet
+from app.core._secrets import hot_pubkey, service_wallet, hot_privkey
 from app.core._blockchain.ton.platform import platform
+from datetime import datetime, timedelta
+from app.core._crypto.signer import Signer
 import subprocess
-import docker
+import json
-from pprint import pprint
 
 
 def get_git_info():
@@ -13,19 +14,47 @@ def get_git_info():
     return branch_name, commit_hash
 
 
-async def s_api_system(request):
+async def s_api_system(request):  # /api/node
-    client = docker.from_env()
-    containers = client.containers.list(all=True)
-    pprint(containers)
-
     return response.json({
         'id': b58encode(hot_pubkey).decode(),
-        'master_address': platform.address.to_string(1, 1, 1),
         'node_address': service_wallet.address.to_string(1, 1, 1),
+        'master_address': platform.address.to_string(1, 1, 1),
         'indexer_height': 0,
+        'services': {
+            service_key: {
+                'status': service['status'],
+                'delay': int((datetime.now() - service['timestamp']).total_seconds()) if service['timestamp'] else -1,
+            }
+            for service_key, service in request.app.ctx.memory.known_states.items()
+        }
     })
 
 
+async def s_api_system_send_status(request):
+    if not request.json:
+        return response.json({'error': 'No data'}, status=400)
+
+    message = request.json.get('message', '')
+    signature = request.json.get('signature', '')
+    if not message or not signature:
+        return response.json({'error': 'No message or signature'}, status=400)
+
+    signer = Signer(hot_privkey)
+    if not signer.verify(message, signature):
+        return response.json({'error': 'Invalid signature'}, status=400)
+
+    message = b58decode(message)
+    message = json.loads(message)
+    request.app.ctx.memory.known_states[
+        message['service']
+    ] = {
+        'status': message['status'],
+        'timestamp': datetime.now(),
+    }
+
+    return response.json({'message': 'Status received'})
+
+
 async def s_api_system_version(request):
     branch_name, commit_hash = get_git_info()
     return response.json({

app/core/_crypto/signer.py

@@ -0,0 +1,29 @@
+from Crypto.PublicKey import ECC
+from Crypto.Hash import SHA256
+from Crypto.Signature import DSS
+import base58
+
+
+class Signer:
+    def __init__(self, seed: bytes):
+        if len(seed) != 32:
+            raise ValueError("Seed must be 32 bytes")
+
+        self.key = ECC.generate(curve='P-256', randfunc=lambda n: seed)
+        self.verifier = DSS.new(self.key, 'fips-186-3')
+
+    def sign(self, data_bytes: bytes) -> str:
+        hash_obj = SHA256.new(data_bytes)
+        signature = self.verifier.sign(hash_obj)
+        signature_str = base58.b58encode(signature).decode()
+        return signature_str
+
+    def verify(self, data_bytes: str, signature: str) -> bool:
+        data_bytes = base58.b58decode(data_bytes)
+        signature_bytes = base58.b58decode(signature)
+        hash_obj = SHA256.new(data_bytes)
+        try:
+            self.verifier.verify(hash_obj, signature_bytes)
+            return True
+        except ValueError:
+            return False

app/core/_secrets.py

@@ -2,7 +2,8 @@ from nacl.bindings import crypto_sign_seed_keypair
 from app.core.active_config import active_config
 from app.core.logger import make_log
 from app.core._blockchain.ton.wallet_v3cr3 import WalletV3CR3
-from os import urandom, getenv
+from os import getenv, urandom
+
 from tonsdk.utils import Address
 
 
@@ -10,7 +11,7 @@ def load_hot_pair():
     hot_seed = active_config.get('private_key')
     if hot_seed is None:
         make_log("HotWallet", "No seed found, generating new one", level='info')
-        hot_seed = urandom(32).hex()
+        hot_seed = urandom(32)
         active_config.set('private_key', hot_seed)
         return load_hot_pair()
 

app/core/_utils/send_status.py

@@ -0,0 +1,29 @@
+from httpx import AsyncClient
+from app.core._config import PROJECT_HOST
+from app.core.logger import make_log
+from app.core._secrets import hot_privkey
+from tonsdk.utils import sign_message
+from base58 import b58encode
+from json import dumps
+from app.core._crypto.signer import Signer
+
+
+async def send_status(service: str, status: str):
+    message = {
+        'service': service,
+        'status': status,
+    }
+    message_bytes = dumps(message).encode()
+    signer = Signer(hot_privkey)
+    message_signature = signer.sign(message_bytes)
+    async with AsyncClient() as client:
+        res = await client.post(
+            f"{PROJECT_HOST}/api/system.sendStatus",
+            json={
+                'message': b58encode(message_bytes).decode(),
+                'signature': message_signature,
+            }
+        )
+        if res.status_code != 200:
+            make_log("send_status", f"Error (service={service}, status={status}, response={res.text})", level='error')
+

app/core/background/indexator_service.py

@@ -1,11 +1,16 @@
-
+from app.core._utils.send_status import send_status
 from app.core.logger import make_log
 import asyncio
 
 
 async def main():
     make_log("Indexator", "Service started", level="info")
+    seqno = 0
+    while True:
 
+        await asyncio.sleep(5)
+        await send_status("indexator", f"working (seqno={seqno})")
+        seqno += 1
 
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()

app/core/background/ton_service.py

@@ -4,6 +4,8 @@ from app.core._config import MY_FUND_ADDRESS
 from app.core.storage import db_session
 from app.core._secrets import service_wallet
 from app.core._blockchain.ton.toncenter import toncenter
+from app.core._utils.send_status import send_status
+
 import asyncio
 import os, sys
 
@@ -48,16 +50,20 @@ async def main():
                 }], sw_seqno_value
             )['message'].to_boc(False)
         )
-
+        make_log("TON", "Withdraw command sent", level="info")
-        sw_seqno_value += 1
+        await asyncio.sleep(10)
 
     while True:
-        make_log("TON", "Service running", level="debug")
+        sw_seqno_value = await get_sw_seqno()
+        make_log("TON", "Service running (seqno+{", level="debug")
         # with db_session() as session:
         #     for stored_content in session.query(StoredContent).filter(StoredContent.uploaded == False).all():
         #         pass
 
         await asyncio.sleep(5)
+        await send_status("ton", f"working (seqno={sw_seqno_value})")
+
+
 
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()

app/core/background/uploader_service.py

@@ -1,3 +1,4 @@
+from app.core._utils.send_status import send_status
 
 from app.core.logger import make_log
 from app.core.storage import db_session
@@ -6,6 +7,7 @@ import asyncio
 
 async def main():
     make_log("Uploader", "Service started", level="info")
+    seqno = 0
     while True:
         # make_log("Uploader", "Service running", level="debug")
         # with db_session() as session:
@@ -13,6 +15,8 @@ async def main():
         #         pass
 
         await asyncio.sleep(5)
+        await send_status("uploader", f"working (seqno={seqno})")
+        seqno += 1
 
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()

app/core/models/memory.py

@@ -3,6 +3,7 @@ from contextlib import asynccontextmanager
 from datetime import datetime, timedelta
 
 from app.core.logger import make_log
+from datetime import datetime
 
 
 class Memory:
@@ -12,6 +13,25 @@ class Memory:
 
         self.transactions_disabled = False
 
+        self.known_states = {
+            "uploader": {
+                "status": "no status",
+                "timestamp": None
+            },
+            "uploader_bot": {
+                "status": "no status",
+                "timestamp": None
+            },
+            "indexator": {
+                "status": "no status",
+                "timestamp": None
+            },
+            "ton_daemon": {
+                "status": "no status",
+                "timestamp": None
+            }
+        }
+
     @asynccontextmanager
     async def transaction(self, desc=""):
         make_log("Memory.transaction", f"Starting transaction; {desc}", level='debug')