busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

chore(semantic): checkpoint remediation progress

Browse Source
This commit is contained in:
busya
2026-03-15 21:08:00 +03:00
parent 15d3141aef
commit 84a2cd5429
25 changed files with 1935 additions and 1559 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/src/core/async_superset_client.py
View File

@@ -4,8 +4,12 @@
# @SEMANTICS: superset, async, client, httpx, dashboards, datasets
# @PURPOSE: Async Superset client for dashboard hot-path requests without blocking FastAPI event loop.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> backend.src.core.superset_client
# @RELATION: DEPENDS_ON -> backend.src.core.utils.async_network.AsyncAPIClient
# @PRE: Environment configuration is valid and Superset endpoint is reachable.
# @POST: Provides non-blocking API access to Superset resources.
# @SIDE_EFFECT: Performs network I/O via httpx.
# @DATA_CONTRACT: Input[Environment] -> Model[dashboard, chart, dataset]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.utils.async_network.AsyncAPIClient]
# @INVARIANT: Async dashboard operations reuse shared auth cache and avoid sync requests in async routes.
# [SECTION: IMPORTS]
@@ -22,9 +26,11 @@ from .utils.async_network import AsyncAPIClient
# [DEF:AsyncSupersetClient:Class]
# @TIER: STANDARD
# @PURPOSE: Async sibling of SupersetClient for dashboard read paths.
class AsyncSupersetClient(SupersetClient):
# [DEF:__init__:Function]
# @TIER: STANDARD
# @PURPOSE: Initialize async Superset client with AsyncAPIClient transport.
# @PRE: env is valid.
# @POST: Client uses async network transport and inherited projection helpers.
@@ -45,6 +51,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:__init__:Function]
# [DEF:aclose:Function]
# @TIER: STANDARD
# @PURPOSE: Close async transport resources.
# @POST: Underlying AsyncAPIClient is closed.
async def aclose(self) -> None:
@@ -52,6 +59,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:aclose:Function]
# [DEF:get_dashboards_page_async:Function]
# @TIER: STANDARD
# @PURPOSE: Fetch one dashboards page asynchronously.
# @POST: Returns total count and page result list.
async def get_dashboards_page_async(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
@@ -85,6 +93,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:get_dashboards_page_async:Function]
# [DEF:get_dashboard_async:Function]
# @TIER: STANDARD
# @PURPOSE: Fetch one dashboard payload asynchronously.
# @POST: Returns raw dashboard payload from Superset API.
async def get_dashboard_async(self, dashboard_id: int) -> Dict:
@@ -94,6 +103,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:get_dashboard_async:Function]
# [DEF:get_chart_async:Function]
# @TIER: STANDARD
# @PURPOSE: Fetch one chart payload asynchronously.
# @POST: Returns raw chart payload from Superset API.
async def get_chart_async(self, chart_id: int) -> Dict:
@@ -103,6 +113,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:get_chart_async:Function]
# [DEF:get_dashboard_detail_async:Function]
# @TIER: STANDARD
# @PURPOSE: Fetch dashboard detail asynchronously with concurrent charts/datasets requests.
# @POST: Returns dashboard detail payload for overview page.
async def get_dashboard_detail_async(self, dashboard_id: int) -> Dict:

16
backend/src/core/auth/repository.py
View File

@@ -4,6 +4,10 @@
# @SEMANTICS: auth, repository, database, user, role, permission
# @PURPOSE: Data access layer for authentication and user preference entities.
# @LAYER: Domain
# @PRE: SQLAlchemy session manager and auth models are available.
# @POST: Provides transactional access to Auth-related database entities.
# @SIDE_EFFECT: Performs database I/O via SQLAlchemy sessions.
# @DATA_CONTRACT: Input[Session] -> Model[User, Role, Permission, UserDashboardPreference]
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.auth]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.profile]
@@ -21,10 +25,12 @@ from ..logger import belief_scope, logger
# [/SECTION]
# [DEF:AuthRepository:Class]
# @TIER: CRITICAL
# @PURPOSE: Encapsulates database operations for authentication-related entities.
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
class AuthRepository:
# [DEF:__init__:Function]
# @TIER: CRITICAL
# @PURPOSE: Bind repository instance to an existing SQLAlchemy session.
# @PRE: db is an initialized sqlalchemy.orm.Session instance.
# @POST: self.db points to the provided session and is used by all repository methods.
@@ -42,6 +48,7 @@ class AuthRepository:
# [/DEF:__init__:Function]
# [DEF:get_user_by_username:Function]
# @TIER: CRITICAL
# @PURPOSE: Retrieve a user entity by unique username.
# @PRE: username is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching User entity when present, otherwise None.
@@ -68,6 +75,7 @@ class AuthRepository:
# [/DEF:get_user_by_username:Function]
# [DEF:get_user_by_id:Function]
# @TIER: CRITICAL
# @PURPOSE: Retrieve a user entity by identifier.
# @PRE: user_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching User entity when present, otherwise None.
@@ -89,6 +97,7 @@ class AuthRepository:
# [/DEF:get_user_by_id:Function]
# [DEF:get_role_by_name:Function]
# @TIER: CRITICAL
# @PURPOSE: Retrieve a role entity by role name.
# @PRE: name is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching Role entity when present, otherwise None.
@@ -100,6 +109,7 @@ class AuthRepository:
# [/DEF:get_role_by_name:Function]
# [DEF:update_last_login:Function]
# @TIER: CRITICAL
# @PURPOSE: Update last_login timestamp for the provided user entity.
# @PRE: user is a managed User instance and self.db is a valid open Session.
# @POST: user.last_login is set to current UTC timestamp and transaction is committed.
@@ -119,6 +129,7 @@ class AuthRepository:
# [/DEF:update_last_login:Function]
# [DEF:get_role_by_id:Function]
# @TIER: CRITICAL
# @PURPOSE: Retrieve a role entity by identifier.
# @PRE: role_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching Role entity when present, otherwise None.
@@ -130,6 +141,7 @@ class AuthRepository:
# [/DEF:get_role_by_id:Function]
# [DEF:get_permission_by_id:Function]
# @TIER: CRITICAL
# @PURPOSE: Retrieve a permission entity by identifier.
# @PRE: perm_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching Permission entity when present, otherwise None.
@@ -141,6 +153,7 @@ class AuthRepository:
# [/DEF:get_permission_by_id:Function]
# [DEF:get_permission_by_resource_action:Function]
# @TIER: CRITICAL
# @PURPOSE: Retrieve a permission entity by resource and action pair.
# @PRE: resource and action are non-empty str values; self.db is a valid open Session.
# @POST: Returns matching Permission entity when present, otherwise None.
@@ -155,6 +168,7 @@ class AuthRepository:
# [/DEF:get_permission_by_resource_action:Function]
# [DEF:get_user_dashboard_preference:Function]
# @TIER: CRITICAL
# @PURPOSE: Retrieve dashboard preference entity owned by specified user.
# @PRE: user_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching UserDashboardPreference entity when present, otherwise None.
@@ -170,6 +184,7 @@ class AuthRepository:
# [/DEF:get_user_dashboard_preference:Function]
# [DEF:save_user_dashboard_preference:Function]
# @TIER: CRITICAL
# @PURPOSE: Persist dashboard preference entity and return refreshed persistent row.
# @PRE: preference is a valid UserDashboardPreference entity and self.db is a valid open Session.
# @POST: preference is committed to DB, refreshed from DB state, and returned.
@@ -192,6 +207,7 @@ class AuthRepository:
# [/DEF:save_user_dashboard_preference:Function]
# [DEF:list_permissions:Function]
# @TIER: CRITICAL
# @PURPOSE: List all permission entities available in storage.
# @PRE: self.db is a valid open Session.
# @POST: Returns list containing all Permission entities visible to the session.

33
backend/src/core/config_manager.py
View File

@@ -4,12 +4,15 @@
# @SEMANTICS: config, manager, persistence, migration, postgresql
# @PURPOSE: Manages application configuration persistence in DB with one-time migration from legacy JSON.
# @LAYER: Domain
# @RELATION: [DEPENDS_ON] ->[ConfigModels]
# @RELATION: [DEPENDS_ON] ->[SessionLocal]
# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]
# @RELATION: [CALLS] ->[logger]
# @RELATION: [CALLS] ->[configure_logger]
# @RELATION: [BINDS_TO] ->[ConfigManager]
# @PRE: Database schema for AppConfigRecord must be initialized.
# @POST: Configuration is loaded into memory and logger is configured.
# @SIDE_EFFECT: Performs DB I/O and may update global logging level.
# @DATA_CONTRACT: Input[json, record] -> Model[AppConfig]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.config_models.AppConfig]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.database.SessionLocal]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.config.AppConfigRecord]
# @RELATION: [CALLS] ->[backend.src.core.logger.logger]
# @RELATION: [CALLS] ->[backend.src.core.logger.configure_logger]
# @INVARIANT: Configuration must always be representable by AppConfig and persisted under global record id.
#
import json
@@ -57,6 +60,7 @@ class ConfigManager:
# [/DEF:__init__:Function]
# [DEF:_default_config:Function]
# @TIER: STANDARD
# @PURPOSE: Build default application configuration fallback.
# @PRE: None.
# @POST: Returns valid AppConfig with empty environments and default storage settings.
@@ -71,6 +75,7 @@ class ConfigManager:
# [/DEF:_default_config:Function]
# [DEF:_sync_raw_payload_from_config:Function]
# @TIER: STANDARD
# @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.
# @PRE: self.config is initialized as AppConfig.
# @POST: self.raw_payload contains AppConfig fields refreshed from self.config.
@@ -85,6 +90,7 @@ class ConfigManager:
# [/DEF:_sync_raw_payload_from_config:Function]
# [DEF:_load_from_legacy_file:Function]
# @TIER: STANDARD
# @PURPOSE: Load legacy JSON configuration for migration fallback path.
# @PRE: self.config_path is initialized.
# @POST: Returns AppConfig from file payload or safe default.
@@ -110,6 +116,7 @@ class ConfigManager:
# [/DEF:_load_from_legacy_file:Function]
# [DEF:_get_record:Function]
# @TIER: STANDARD
# @PURPOSE: Resolve global configuration record from DB.
# @PRE: session is an active SQLAlchemy Session.
# @POST: Returns record when present, otherwise None.
@@ -121,6 +128,7 @@ class ConfigManager:
# [/DEF:_get_record:Function]
# [DEF:_load_config:Function]
# @TIER: STANDARD
# @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON.
# @PRE: SessionLocal factory is available and AppConfigRecord schema is accessible.
# @POST: Returns valid AppConfig and closes opened DB session.
@@ -152,6 +160,7 @@ class ConfigManager:
# [/DEF:_load_config:Function]
# [DEF:_save_config_to_db:Function]
# @TIER: STANDARD
# @PURPOSE: Persist provided AppConfig into the global DB configuration record.
# @PRE: config is AppConfig; session is either None or an active Session.
# @POST: Global DB record payload equals config.model_dump() when commit succeeds.
@@ -186,6 +195,7 @@ class ConfigManager:
# [/DEF:_save_config_to_db:Function]
# [DEF:save:Function]
# @TIER: STANDARD
# @PURPOSE: Persist current in-memory configuration state.
# @PRE: self.config is initialized.
# @POST: Current self.config is written to DB global record.
@@ -197,6 +207,7 @@ class ConfigManager:
# [/DEF:save:Function]
# [DEF:get_config:Function]
# @TIER: STANDARD
# @PURPOSE: Return current in-memory configuration snapshot.
# @PRE: self.config is initialized.
# @POST: Returns AppConfig reference stored in manager.
@@ -208,6 +219,7 @@ class ConfigManager:
# [/DEF:get_config:Function]
# [DEF:get_payload:Function]
# @TIER: STANDARD
# @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema.
# @PRE: Manager state is initialized.
# @POST: Returns dict payload with current AppConfig fields synchronized.
@@ -219,6 +231,7 @@ class ConfigManager:
# [/DEF:get_payload:Function]
# [DEF:save_config:Function]
# @TIER: STANDARD
# @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict.
# @PRE: config is AppConfig or dict compatible with AppConfig core schema.
# @POST: self.config and self.raw_payload are synchronized and persisted to DB.
@@ -240,6 +253,7 @@ class ConfigManager:
# [/DEF:save_config:Function]
# [DEF:update_global_settings:Function]
# @TIER: STANDARD
# @PURPOSE: Replace global settings and persist the resulting configuration.
# @PRE: settings is GlobalSettings.
# @POST: self.config.settings equals provided settings and DB state is updated.
@@ -258,6 +272,7 @@ class ConfigManager:
# [/DEF:update_global_settings:Function]
# [DEF:validate_path:Function]
# @TIER: STANDARD
# @PURPOSE: Validate that path exists and is writable, creating it when absent.
# @PRE: path is a string path candidate.
# @POST: Returns (True, msg) for writable path, else (False, reason).
@@ -279,6 +294,7 @@ class ConfigManager:
# [/DEF:validate_path:Function]
# [DEF:get_environments:Function]
# @TIER: STANDARD
# @PURPOSE: Return all configured environments.
# @PRE: self.config is initialized.
# @POST: Returns list of Environment models from current configuration.
@@ -290,6 +306,7 @@ class ConfigManager:
# [/DEF:get_environments:Function]
# [DEF:has_environments:Function]
# @TIER: STANDARD
# @PURPOSE: Check whether at least one environment exists in configuration.
# @PRE: self.config is initialized.
# @POST: Returns True iff environment list length is greater than zero.
@@ -301,6 +318,7 @@ class ConfigManager:
# [/DEF:has_environments:Function]
# [DEF:get_environment:Function]
# @TIER: STANDARD
# @PURPOSE: Resolve a configured environment by identifier.
# @PRE: env_id is string identifier.
# @POST: Returns matching Environment when found; otherwise None.
@@ -315,6 +333,7 @@ class ConfigManager:
# [/DEF:get_environment:Function]
# [DEF:add_environment:Function]
# @TIER: STANDARD
# @PURPOSE: Upsert environment by id into configuration and persist.
# @PRE: env is Environment.
# @POST: Configuration contains provided env id with new payload persisted.
@@ -333,6 +352,7 @@ class ConfigManager:
# [/DEF:add_environment:Function]
# [DEF:update_environment:Function]
# @TIER: STANDARD
# @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior.
# @PRE: env_id is non-empty string and updated_env is Environment.
# @POST: Returns True and persists update when target exists; else returns False.
@@ -362,6 +382,7 @@ class ConfigManager:
# [/DEF:update_environment:Function]
# [DEF:delete_environment:Function]
# @TIER: STANDARD
# @PURPOSE: Delete environment by id and persist when deletion occurs.
# @PRE: env_id is non-empty string.
# @POST: Environment is removed when present; otherwise configuration is unchanged.

26
backend/src/core/database.py
View File

@@ -4,9 +4,9 @@
# @SEMANTICS: database, postgresql, sqlalchemy, session, persistence
# @PURPOSE: Configures database connection and session management (PostgreSQL-first).
# @LAYER: Core
# @RELATION: DEPENDS_ON -> sqlalchemy
# @RELATION: DEPENDS_ON -> backend.src.models.mapping
# @RELATION: DEPENDS_ON -> backend.src.core.auth.config
# @RELATION: DEPENDS_ON ->[sqlalchemy]
# @RELATION: DEPENDS_ON ->[backend.src.models.mapping]
# @RELATION: DEPENDS_ON ->[backend.src.core.auth.config]
#
# @INVARIANT: A single engine instance is used for the entire application.
@@ -31,11 +31,13 @@ from pathlib import Path
# [/SECTION]
# [DEF:BASE_DIR:Variable]
# @TIER: TRIVIAL
# @PURPOSE: Base directory for the backend.
BASE_DIR = Path(__file__).resolve().parent.parent.parent
# [/DEF:BASE_DIR:Variable]
# [DEF:DATABASE_URL:Constant]
# @TIER: TRIVIAL
# @PURPOSE: URL for the main application database.
DEFAULT_POSTGRES_URL = os.getenv(
"POSTGRES_URL",
@@ -45,34 +47,39 @@ DATABASE_URL = os.getenv("DATABASE_URL", DEFAULT_POSTGRES_URL)
# [/DEF:DATABASE_URL:Constant]
# [DEF:TASKS_DATABASE_URL:Constant]
# @TIER: TRIVIAL
# @PURPOSE: URL for the tasks execution database.
# Defaults to DATABASE_URL to keep task logs in the same PostgreSQL instance.
TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", DATABASE_URL)
# [/DEF:TASKS_DATABASE_URL:Constant]
# [DEF:AUTH_DATABASE_URL:Constant]
# @TIER: TRIVIAL
# @PURPOSE: URL for the authentication database.
AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL)
# [/DEF:AUTH_DATABASE_URL:Constant]
# [DEF:engine:Variable]
# @TIER: TRIVIAL
# @PURPOSE: SQLAlchemy engine for mappings database.
# @SIDE_EFFECT: Creates database engine and manages connection pool.
def _build_engine(db_url: str):
with belief_scope("_build_engine"):
if db_url.startswith("sqlite"):
return create_engine(db_url, connect_args={"check_same_thread": False})
return create_engine(db_url, pool_pre_ping=True)
# @PURPOSE: SQLAlchemy engine for mappings database.
engine = _build_engine(DATABASE_URL)
# [/DEF:engine:Variable]
# [DEF:tasks_engine:Variable]
# @TIER: TRIVIAL
# @PURPOSE: SQLAlchemy engine for tasks database.
tasks_engine = _build_engine(TASKS_DATABASE_URL)
# [/DEF:tasks_engine:Variable]
# [DEF:auth_engine:Variable]
# @TIER: TRIVIAL
# @PURPOSE: SQLAlchemy engine for authentication database.
auth_engine = _build_engine(AUTH_DATABASE_URL)
# [/DEF:auth_engine:Variable]
@@ -99,6 +106,7 @@ AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_eng
# [/DEF:AuthSessionLocal:Class]
# [DEF:_ensure_user_dashboard_preferences_columns:Function]
# @TIER: STANDARD
# @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table.
# @PRE: bind_engine points to application database where profile table is stored.
# @POST: Missing columns are added without data loss.
@@ -165,6 +173,7 @@ def _ensure_user_dashboard_preferences_columns(bind_engine):
# [DEF:_ensure_user_dashboard_preferences_health_columns:Function]
# @TIER: STANDARD
# @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table (health fields).
def _ensure_user_dashboard_preferences_health_columns(bind_engine):
with belief_scope("_ensure_user_dashboard_preferences_health_columns"):
@@ -208,6 +217,7 @@ def _ensure_user_dashboard_preferences_health_columns(bind_engine):
# [DEF:_ensure_llm_validation_results_columns:Function]
# @TIER: STANDARD
# @PURPOSE: Applies additive schema upgrades for llm_validation_results table.
def _ensure_llm_validation_results_columns(bind_engine):
with belief_scope("_ensure_llm_validation_results_columns"):
@@ -247,6 +257,7 @@ def _ensure_llm_validation_results_columns(bind_engine):
# [DEF:_ensure_git_server_configs_columns:Function]
# @TIER: STANDARD
# @PURPOSE: Applies additive schema upgrades for git_server_configs table.
# @PRE: bind_engine points to application database.
# @POST: Missing columns are added without data loss.
@@ -284,6 +295,7 @@ def _ensure_git_server_configs_columns(bind_engine):
# [DEF:ensure_connection_configs_table:Function]
# @TIER: STANDARD
# @PURPOSE: Ensures the external connection registry table exists in the main database.
# @PRE: bind_engine points to the application database.
# @POST: connection_configs table exists without dropping existing data.
@@ -301,6 +313,7 @@ def ensure_connection_configs_table(bind_engine):
# [DEF:init_db:Function]
# @TIER: STANDARD
# @PURPOSE: Initializes the database by creating all tables.
# @PRE: engine, tasks_engine and auth_engine are initialized.
# @POST: Database tables created in all databases.
@@ -318,6 +331,7 @@ def init_db():
# [/DEF:init_db:Function]
# [DEF:get_db:Function]
# @TIER: STANDARD
# @PURPOSE: Dependency for getting a database session.
# @PRE: SessionLocal is initialized.
# @POST: Session is closed after use.
@@ -332,6 +346,7 @@ def get_db():
# [/DEF:get_db:Function]
# [DEF:get_tasks_db:Function]
# @TIER: STANDARD
# @PURPOSE: Dependency for getting a tasks database session.
# @PRE: TasksSessionLocal is initialized.
# @POST: Session is closed after use.
@@ -346,6 +361,7 @@ def get_tasks_db():
# [/DEF:get_tasks_db:Function]
# [DEF:get_auth_db:Function]
# @TIER: STANDARD
# @PURPOSE: Dependency for getting an authentication database session.
# @PRE: AuthSessionLocal is initialized.
# @POST: Session is closed after use.

31
backend/src/core/task_manager/manager.py
View File

@@ -3,7 +3,12 @@
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
# @LAYER: Core
# @RELATION: Depends on PluginLoader to get plugin instances. It is used by the API layer to create and query tasks.
# @PRE: Plugin loader and database sessions are initialized.
# @POST: Orchestrates task execution and persistence.
# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB.
# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.plugin_loader]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.persistence]
# @INVARIANT: Task IDs are unique.
# @CONSTRAINT: Must use belief_scope for logging.
# @TEST_CONTRACT: TaskManagerModule -> {
@@ -33,9 +38,9 @@ from ..logger import logger, belief_scope, should_log_task_level
# [/SECTION]
# [DEF:TaskManager:Class]
# @TIER: CRITICAL
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
# @TIER: CRITICAL
# @INVARIANT: Task IDs are unique within the registry.
# @INVARIANT: Each task has exactly one status at any time.
# @INVARIANT: Log entries are never deleted after being added to a task.
@@ -62,6 +67,7 @@ class TaskManager:
LOG_FLUSH_INTERVAL = 2.0
# [DEF:__init__:Function]
# @TIER: CRITICAL
# @PURPOSE: Initialize the TaskManager with dependencies.
# @PRE: plugin_loader is initialized.
# @POST: TaskManager is ready to accept tasks.
@@ -95,6 +101,7 @@ class TaskManager:
# [/DEF:__init__:Function]
# [DEF:_flusher_loop:Function]
# @TIER: STANDARD
# @PURPOSE: Background thread that periodically flushes log buffer to database.
# @PRE: TaskManager is initialized.
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
@@ -106,6 +113,7 @@ class TaskManager:
# [/DEF:_flusher_loop:Function]
# [DEF:_flush_logs:Function]
# @TIER: STANDARD
# @PURPOSE: Flush all buffered logs to the database.
# @PRE: None.
# @POST: All buffered logs are written to task_logs table.
@@ -132,6 +140,7 @@ class TaskManager:
# [/DEF:_flush_logs:Function]
# [DEF:_flush_task_logs:Function]
# @TIER: STANDARD
# @PURPOSE: Flush logs for a specific task immediately.
# @PRE: task_id exists.
# @POST: Task's buffered logs are written to database.
@@ -150,6 +159,7 @@ class TaskManager:
# [/DEF:_flush_task_logs:Function]
# [DEF:create_task:Function]
# @TIER: STANDARD
# @PURPOSE: Creates and queues a new task for execution.
# @PRE: Plugin with plugin_id exists. Params are valid.
# @POST: Task is created, added to registry, and scheduled for execution.
@@ -179,6 +189,7 @@ class TaskManager:
# [/DEF:create_task:Function]
# [DEF:_run_task:Function]
# @TIER: STANDARD
# @PURPOSE: Internal method to execute a task with TaskContext support.
# @PRE: Task exists in registry.
# @POST: Task is executed, status updated to SUCCESS or FAILED.
@@ -246,6 +257,7 @@ class TaskManager:
# [/DEF:_run_task:Function]
# [DEF:resolve_task:Function]
# @TIER: STANDARD
# @PURPOSE: Resumes a task that is awaiting mapping.
# @PRE: Task exists and is in AWAITING_MAPPING state.
# @POST: Task status updated to RUNNING, params updated, execution resumed.
@@ -270,6 +282,7 @@ class TaskManager:
# [/DEF:resolve_task:Function]
# [DEF:wait_for_resolution:Function]
# @TIER: STANDARD
# @PURPOSE: Pauses execution and waits for a resolution signal.
# @PRE: Task exists.
# @POST: Execution pauses until future is set.
@@ -292,6 +305,7 @@ class TaskManager:
# [/DEF:wait_for_resolution:Function]
# [DEF:wait_for_input:Function]
# @TIER: STANDARD
# @PURPOSE: Pauses execution and waits for user input.
# @PRE: Task exists.
# @POST: Execution pauses until future is set via resume_task_with_password.
@@ -313,6 +327,7 @@ class TaskManager:
# [/DEF:wait_for_input:Function]
# [DEF:get_task:Function]
# @TIER: STANDARD
# @PURPOSE: Retrieves a task by its ID.
# @PRE: task_id is a string.
# @POST: Returns Task object or None.
@@ -324,6 +339,7 @@ class TaskManager:
# [/DEF:get_task:Function]
# [DEF:get_all_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Retrieves all registered tasks.
# @PRE: None.
# @POST: Returns list of all Task objects.
@@ -334,6 +350,7 @@ class TaskManager:
# [/DEF:get_all_tasks:Function]
# [DEF:get_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Retrieves tasks with pagination and optional status filter.
# @PRE: limit and offset are non-negative integers.
# @POST: Returns a list of tasks sorted by start_time descending.
@@ -374,6 +391,7 @@ class TaskManager:
# [/DEF:get_tasks:Function]
# [DEF:get_task_logs:Function]
# @TIER: STANDARD
# @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).
# @PRE: task_id is a string.
# @POST: Returns list of LogEntry or TaskLog objects.
@@ -406,6 +424,7 @@ class TaskManager:
# [/DEF:get_task_logs:Function]
# [DEF:get_task_log_stats:Function]
# @TIER: STANDARD
# @PURPOSE: Get statistics about logs for a task.
# @PRE: task_id is a valid task ID.
# @POST: Returns LogStats with counts by level and source.
@@ -417,6 +436,7 @@ class TaskManager:
# [/DEF:get_task_log_stats:Function]
# [DEF:get_task_log_sources:Function]
# @TIER: STANDARD
# @PURPOSE: Get unique sources for a task's logs.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of unique source strings.
@@ -428,6 +448,7 @@ class TaskManager:
# [/DEF:get_task_log_sources:Function]
# [DEF:_add_log:Function]
# @TIER: STANDARD
# @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.
# @PRE: Task exists.
# @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).
@@ -480,6 +501,7 @@ class TaskManager:
# [/DEF:_add_log:Function]
# [DEF:subscribe_logs:Function]
# @TIER: STANDARD
# @PURPOSE: Subscribes to real-time logs for a task.
# @PRE: task_id is a string.
# @POST: Returns an asyncio.Queue for log entries.
@@ -495,6 +517,7 @@ class TaskManager:
# [/DEF:subscribe_logs:Function]
# [DEF:unsubscribe_logs:Function]
# @TIER: STANDARD
# @PURPOSE: Unsubscribes from real-time logs for a task.
# @PRE: task_id is a string, queue is asyncio.Queue.
# @POST: Queue removed from subscribers.
@@ -510,6 +533,7 @@ class TaskManager:
# [/DEF:unsubscribe_logs:Function]
# [DEF:load_persisted_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Load persisted tasks using persistence service.
# @PRE: None.
# @POST: Persisted tasks loaded into self.tasks.
@@ -522,6 +546,7 @@ class TaskManager:
# [/DEF:load_persisted_tasks:Function]
# [DEF:await_input:Function]
# @TIER: STANDARD
# @PURPOSE: Transition a task to AWAITING_INPUT state with input request.
# @PRE: Task exists and is in RUNNING state.
# @POST: Task status changed to AWAITING_INPUT, input_request set, persisted.
@@ -544,6 +569,7 @@ class TaskManager:
# [/DEF:await_input:Function]
# [DEF:resume_task_with_password:Function]
# @TIER: STANDARD
# @PURPOSE: Resume a task that is awaiting input with provided passwords.
# @PRE: Task exists and is in AWAITING_INPUT state.
# @POST: Task status changed to RUNNING, passwords injected, task resumed.
@@ -573,6 +599,7 @@ class TaskManager:
# [/DEF:resume_task_with_password:Function]
# [DEF:clear_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Clears tasks based on status filter (also deletes associated logs).
# @PRE: status is Optional[TaskStatus].
# @POST: Tasks matching filter (or all non-active) cleared from registry and database.

20
backend/src/core/task_manager/persistence.py
View File

@@ -3,7 +3,12 @@
# @SEMANTICS: persistence, sqlite, sqlalchemy, task, storage
# @PURPOSE: Handles the persistence of tasks using SQLAlchemy and the tasks.db database.
# @LAYER: Core
# @RELATION: Used by TaskManager to save and load tasks.
# @PRE: Tasks database must be initialized with TaskRecord and TaskLogRecord schemas.
# @POST: Provides reliable storage and retrieval for task metadata and logs.
# @SIDE_EFFECT: Performs database I/O on tasks.db.
# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.database.TasksSessionLocal]
# @INVARIANT: Database schema must match the TaskRecord model structure.
# [SECTION: IMPORTS]
@@ -118,6 +123,7 @@ class TaskPersistenceService:
# [/DEF:_resolve_environment_id:Function]
# [DEF:__init__:Function]
# @TIER: STANDARD
# @PURPOSE: Initializes the persistence service.
# @PRE: None.
# @POST: Service is ready.
@@ -128,6 +134,7 @@ class TaskPersistenceService:
# [/DEF:__init__:Function]
# [DEF:persist_task:Function]
# @TIER: STANDARD
# @PURPOSE: Persists or updates a single task in the database.
# @PRE: isinstance(task, Task)
# @POST: Task record created or updated in database.
@@ -190,6 +197,7 @@ class TaskPersistenceService:
# [/DEF:persist_task:Function]
# [DEF:persist_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Persists multiple tasks.
# @PRE: isinstance(tasks, list)
# @POST: All tasks in list are persisted.
@@ -201,6 +209,7 @@ class TaskPersistenceService:
# [/DEF:persist_tasks:Function]
# [DEF:load_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Loads tasks from the database.
# @PRE: limit is an integer.
# @POST: Returns list of Task objects.
@@ -255,6 +264,7 @@ class TaskPersistenceService:
# [/DEF:load_tasks:Function]
# [DEF:delete_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Deletes specific tasks from the database.
# @PRE: task_ids is a list of strings.
# @POST: Specified task records deleted from database.
@@ -277,9 +287,9 @@ class TaskPersistenceService:
# [/DEF:TaskPersistenceService:Class]
# [DEF:TaskLogPersistenceService:Class]
# @TIER: CRITICAL
# @SEMANTICS: persistence, service, database, log, sqlalchemy
# @PURPOSE: Provides methods to save and query task logs from the task_logs table.
# @TIER: CRITICAL
# @RELATION: DEPENDS_ON -> TaskLogRecord
# @INVARIANT: Log entries are batch-inserted for performance.
#
@@ -311,6 +321,7 @@ class TaskLogPersistenceService:
# [/DEF:__init__:Function]
# [DEF:add_logs:Function]
# @TIER: STANDARD
# @PURPOSE: Batch insert log entries for a task.
# @PRE: logs is a list of LogEntry objects.
# @POST: All logs inserted into task_logs table.
@@ -342,6 +353,7 @@ class TaskLogPersistenceService:
# [/DEF:add_logs:Function]
# [DEF:get_logs:Function]
# @TIER: STANDARD
# @PURPOSE: Query logs for a task with filtering and pagination.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of TaskLog objects matching filters.
@@ -394,6 +406,7 @@ class TaskLogPersistenceService:
# [/DEF:get_logs:Function]
# [DEF:get_log_stats:Function]
# @TIER: STANDARD
# @PURPOSE: Get statistics about logs for a task.
# @PRE: task_id is a valid task ID.
# @POST: Returns LogStats with counts by level and source.
@@ -439,6 +452,7 @@ class TaskLogPersistenceService:
# [/DEF:get_log_stats:Function]
# [DEF:get_sources:Function]
# @TIER: STANDARD
# @PURPOSE: Get unique sources for a task's logs.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of unique source strings.
@@ -458,6 +472,7 @@ class TaskLogPersistenceService:
# [/DEF:get_sources:Function]
# [DEF:delete_logs_for_task:Function]
# @TIER: STANDARD
# @PURPOSE: Delete all logs for a specific task.
# @PRE: task_id is a valid task ID.
# @POST: All logs for the task are deleted.
@@ -479,6 +494,7 @@ class TaskLogPersistenceService:
# [/DEF:delete_logs_for_task:Function]
# [DEF:delete_logs_for_tasks:Function]
# @TIER: STANDARD
# @PURPOSE: Delete all logs for multiple tasks.
# @PRE: task_ids is a list of task IDs.
# @POST: All logs for the tasks are deleted.