init

backend/core/settings.py

@@ -0,0 +1,48 @@
+import os
+from pathlib import Path
+from functools import lru_cache
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# ─── Paths ───────────────────────────────────────────────
+ROOT_DIR               = Path(__file__).resolve().parent.parent
+DATA_DIR               = ROOT_DIR / "data"
+TMP_DIR                = Path("/tmp")
+DOWNLOAD_DIR           = DATA_DIR / "downloads"
+USERS_DIR              = DATA_DIR / "users"
+ENV                    = os.getenv("APP_ENV", "development")
+PROXY_LIST_FILE        = Path(os.getenv("PROXY_LIST_FILE", ".50.txt"))
+# ─── Additional ───────────────────────────────────────────────
+PROXY_USERNAME         = os.getenv("PROXY_USERNAME")
+PROXY_PASSWORD         = os.getenv("PROXY_PASSWORD")
+# ─── DB and SQLAlchemy ───────────────────────────────────────────
+SQLALCHEMY_DATABASE_URI= os.getenv("DATABASE_URL") or f"sqlite:///{DATA_DIR / 'local.db'}"
+DB_POOL_SIZE           = int(os.getenv("DB_POOL_SIZE", 20))
+DB_ECHO                = bool(os.getenv("DB_ECHO", False))
+# ─── Concurrency ───────────────────────────────────────────────
+CPU_COUNT              = os.cpu_count() or 2
+THREADS_MAX            = min(32, CPU_COUNT * 4)
+PROCS_MAX              = min(CPU_COUNT, 4)
+PER_IP_CONCURRENCY     = int(os.getenv("PER_IP_CONCURRENCY", 2))
+# ─── Cache and Tuning knobs ──────────────────────────────────────
+FORMAT_CACHE_TTL_SEC   = int(os.getenv("FORMAT_CACHE_TTL_SEC",   8_000))
+DOWNLOAD_CACHE_TTL_SEC = int(os.getenv("DOWNLOAD_CACHE_TTL_SEC", 86_400)) # 24h
+PARALLEL_CHUNK_MB      = int(os.getenv("PARALLEL_CHUNK_MB",      2))
+MAX_CONCURRENT_FRAG    = int(os.getenv("MAX_CONCURRENT_FRAG",    4))
+ARIA2C_THRESHOLD_MB    = int(os.getenv("ARIA2C_THRESHOLD_MB",    512))
+MIN_SCORE              = float(os.getenv("PROXY_MIN_SCORE", "0.05"))
+MAX_IN_USE             = int(os.getenv("PROXY_CONCURRENCY_LIMIT", "4"))
+FAIL_COOLDOWN_SEC      = int(os.getenv("PROXY_FAIL_COOLDOWN", "600"))
+_MAX_LOGIN_FAILS       = int(os.getenv("MAX_LOGIN_FAILS", "12"))
+_MAX_INVALID_URLS      = int(os.getenv("MAX_INVALID_URLS", "20"))
+_WINDOW_MINUTES        = int(os.getenv("WINDOW_MINUTES", "60"))
+# ─── Logging ───────────────────────────────────────────────────
+LOG_LEVEL              = os.getenv("LOG_LEVEL", "INFO")
+# ─── db ───────────────────────────────────────────────────
+SKIP_SCHEMA_BOOTSTRAP  = int(os.getenv("SKIP_SCHEMA_BOOTSTRAP","0"))
+@lru_cache
+def ensure_dirs() -> None:
+    for p in (DATA_DIR, USERS_DIR, DOWNLOAD_DIR):
+        p.mkdir(parents=True, exist_ok=True)
+ensure_dirs()