from datetime import datetime import logging import time import httpx import threading import os PROJSCALE_APP_NAME = os.getenv('APP_PROJSCALE_NAME', 'my-uploader') LOGS_DIRECTORY = os.getenv('APP_LOGS_DIRECTORY', 'logs') os.makedirs(LOGS_DIRECTORY, exist_ok=True) FORMAT_STRING = '%(asctime)s - %(levelname)s – %(pathname)s – %(funcName)s – %(lineno)d - %(message)s' def push_logs_to_loki(log_entries: list[int, str], attempt: int = 0): try: response = httpx.post( "https://loki-api.projscale.dev/loki/api/v1/push", json={ 'streams': [ { 'stream': { 'label': 'externalLogs', 'appName': PROJSCALE_APP_NAME }, 'values': log_entries } ] }, headers={ 'Content-Type': 'application/json' } ) assert response.status_code == 204, f"Invalid HTTP status code" except BaseException as e: if attempt < 3: time.sleep(3) push_logs_to_loki(log_entries, attempt + 1) else: print(f"Failed to push logs to Loki: {e}") class ProjscaleLoggingHandler(logging.Handler): def __init__(self): super().__init__() self.setFormatter(logging.Formatter(FORMAT_STRING)) self.logs_cache = [] self.logs_pushed = 0 def emit(self, record): log_entry = self.format(record) self.logs_cache.append([str(int(time.time() * 1e9)), log_entry]) if ((time.time() - self.logs_pushed) > 5) or (len(self.logs_cache) > 5_000): threading.Thread(target=push_logs_to_loki, args=(self.logs_cache,)).start() self.logs_cache = [] self.logs_pushed = time.time() logger = logging.getLogger('uploaderMY') projscale_handler = ProjscaleLoggingHandler() projscale_handler.setLevel(logging.DEBUG) logger.addHandler(projscale_handler) log_filepath = f"{LOGS_DIRECTORY}/{datetime.now().strftime('%Y-%m-%d_%H')}.log" file_handler = logging.FileHandler(log_filepath) file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(logging.Formatter(FORMAT_STRING)) logger.addHandler(file_handler) if os.getenv('APP_ENABLE_STDOUT_LOGS', '0') == '1': stdout_handler = logging.StreamHandler() stdout_handler.setLevel(logging.DEBUG) stdout_handler.setFormatter(logging.Formatter(FORMAT_STRING)) logger.addHandler(stdout_handler) logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("sanic").setLevel(logging.WARNING) logger.setLevel(logging.DEBUG)