busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor(semantics): migrate legacy @TIER to @COMPLEXITY annotations

Browse Source 
- Replaced @TIER: TRIVIAL with @COMPLEXITY: 1
- Replaced @TIER: STANDARD with @COMPLEXITY: 3
- Replaced @TIER: CRITICAL with @COMPLEXITY: 5
- Manually elevated specific critical/complex components to levels 2 and 4
- Ignored legacy, specs, and node_modules directories
- Updated generated semantic map
This commit is contained in:
busya
2026-03-16 10:06:44 +03:00
parent 321e0eb2db
commit 274510fc38
321 changed files with 30101 additions and 58483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/core/__tests__/test_config_manager_compat.py
View File

@@ -1,5 +1,5 @@
# [DEF:backend.src.core.__tests__.test_config_manager_compat:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: config-manager, compatibility, payload, tests
# @PURPOSE: Verifies ConfigManager compatibility wrappers preserve legacy payload sections.
# @LAYER: Domain

2
backend/src/core/__tests__/test_superset_profile_lookup.py
View File

@@ -1,5 +1,5 @@
# [DEF:backend.src.core.__tests__.test_superset_profile_lookup:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: tests, superset, profile, lookup, fallback, sorting
# @PURPOSE: Verifies Superset profile lookup adapter payload normalization and fallback error precedence.
# @LAYER: Domain

2
backend/src/core/__tests__/test_throttled_scheduler.py
View File

@@ -3,7 +3,7 @@ from datetime import time, date, datetime, timedelta
from src.core.scheduler import ThrottledSchedulerConfigurator
# [DEF:test_throttled_scheduler:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for ThrottledSchedulerConfigurator distribution logic.
def test_calculate_schedule_even_distribution():

16
backend/src/core/async_superset_client.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.async_superset_client:Module]
#
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: superset, async, client, httpx, dashboards, datasets
# @PURPOSE: Async Superset client for dashboard hot-path requests without blocking FastAPI event loop.
# @LAYER: Core
@@ -26,14 +26,14 @@ from .utils.async_network import AsyncAPIClient
# [DEF:backend.src.core.async_superset_client.AsyncSupersetClient:Class]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Async sibling of SupersetClient for dashboard read paths.
# @RELATION: [INHERITS] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.utils.async_network.AsyncAPIClient]
# @RELATION: [CALLS] ->[backend.src.core.utils.async_network.AsyncAPIClient.request]
class AsyncSupersetClient(SupersetClient):
# [DEF:backend.src.core.async_superset_client.AsyncSupersetClient.__init__:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Initialize async Superset client with AsyncAPIClient transport.
# @PRE: env is valid Environment instance.
# @POST: Client uses async network transport and inherited projection helpers.
@@ -55,7 +55,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:backend.src.core.async_superset_client.AsyncSupersetClient.__init__:Function]
# [DEF:backend.src.core.async_superset_client.AsyncSupersetClient.aclose:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Close async transport resources.
# @POST: Underlying AsyncAPIClient is closed.
# @SIDE_EFFECT: Closes network sockets.
@@ -64,7 +64,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:backend.src.core.async_superset_client.AsyncSupersetClient.aclose:Function]
# [DEF:backend.src.core.async_superset_client.AsyncSupersetClient.get_dashboards_page_async:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetch one dashboards page asynchronously.
# @POST: Returns total count and page result list.
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
@@ -99,7 +99,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:get_dashboards_page_async:Function]
# [DEF:get_dashboard_async:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetch one dashboard payload asynchronously.
# @POST: Returns raw dashboard payload from Superset API.
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict]
@@ -110,7 +110,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:get_dashboard_async:Function]
# [DEF:get_chart_async:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetch one chart payload asynchronously.
# @POST: Returns raw chart payload from Superset API.
# @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict]
@@ -121,7 +121,7 @@ class AsyncSupersetClient(SupersetClient):
# [/DEF:get_chart_async:Function]
# [DEF:get_dashboard_detail_async:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetch dashboard detail asynchronously with concurrent charts/datasets requests.
# @POST: Returns dashboard detail payload for overview page.
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict]

2
backend/src/core/auth/__tests__/test_auth.py
View File

@@ -1,5 +1,5 @@
# [DEF:test_auth:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for authentication module
# @LAYER: Domain
# @RELATION: VERIFIES -> src.core.auth

2
backend/src/core/auth/jwt.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.auth.jwt:Module]
#
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: jwt, token, session, auth
# @PURPOSE: JWT token generation and validation logic.
# @LAYER: Core

2
backend/src/core/auth/logger.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.auth.logger:Module]
#
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: auth, logger, audit, security
# @PURPOSE: Audit logging for security-related events.
# @LAYER: Core

26
backend/src/core/auth/repository.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.auth.repository:Module]
#
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: auth, repository, database, user, role, permission
# @PURPOSE: Data access layer for authentication and user preference entities.
# @LAYER: Domain
@@ -25,12 +25,12 @@ from ..logger import belief_scope, logger
# [/SECTION]
# [DEF:AuthRepository:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Encapsulates database operations for authentication-related entities.
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
class AuthRepository:
# [DEF:__init__:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Bind repository instance to an existing SQLAlchemy session.
# @PRE: db is an initialized sqlalchemy.orm.Session instance.
# @POST: self.db points to the provided session and is used by all repository methods.
@@ -48,7 +48,7 @@ class AuthRepository:
# [/DEF:__init__:Function]
# [DEF:get_user_by_username:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Retrieve a user entity by unique username.
# @PRE: username is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching User entity when present, otherwise None.
@@ -75,7 +75,7 @@ class AuthRepository:
# [/DEF:get_user_by_username:Function]
# [DEF:get_user_by_id:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Retrieve a user entity by identifier.
# @PRE: user_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching User entity when present, otherwise None.
@@ -97,7 +97,7 @@ class AuthRepository:
# [/DEF:get_user_by_id:Function]
# [DEF:get_role_by_name:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Retrieve a role entity by role name.
# @PRE: name is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching Role entity when present, otherwise None.
@@ -109,7 +109,7 @@ class AuthRepository:
# [/DEF:get_role_by_name:Function]
# [DEF:update_last_login:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Update last_login timestamp for the provided user entity.
# @PRE: user is a managed User instance and self.db is a valid open Session.
# @POST: user.last_login is set to current UTC timestamp and transaction is committed.
@@ -129,7 +129,7 @@ class AuthRepository:
# [/DEF:update_last_login:Function]
# [DEF:get_role_by_id:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Retrieve a role entity by identifier.
# @PRE: role_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching Role entity when present, otherwise None.
@@ -141,7 +141,7 @@ class AuthRepository:
# [/DEF:get_role_by_id:Function]
# [DEF:get_permission_by_id:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Retrieve a permission entity by identifier.
# @PRE: perm_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching Permission entity when present, otherwise None.
@@ -153,7 +153,7 @@ class AuthRepository:
# [/DEF:get_permission_by_id:Function]
# [DEF:get_permission_by_resource_action:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Retrieve a permission entity by resource and action pair.
# @PRE: resource and action are non-empty str values; self.db is a valid open Session.
# @POST: Returns matching Permission entity when present, otherwise None.
@@ -168,7 +168,7 @@ class AuthRepository:
# [/DEF:get_permission_by_resource_action:Function]
# [DEF:get_user_dashboard_preference:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Retrieve dashboard preference entity owned by specified user.
# @PRE: user_id is a non-empty str and self.db is a valid open Session.
# @POST: Returns matching UserDashboardPreference entity when present, otherwise None.
@@ -184,7 +184,7 @@ class AuthRepository:
# [/DEF:get_user_dashboard_preference:Function]
# [DEF:save_user_dashboard_preference:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Persist dashboard preference entity and return refreshed persistent row.
# @PRE: preference is a valid UserDashboardPreference entity and self.db is a valid open Session.
# @POST: preference is committed to DB, refreshed from DB state, and returned.
@@ -207,7 +207,7 @@ class AuthRepository:
# [/DEF:save_user_dashboard_preference:Function]
# [DEF:list_permissions:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: List all permission entities available in storage.
# @PRE: self.db is a valid open Session.
# @POST: Returns list containing all Permission entities visible to the session.

40
backend/src/core/config_manager.py
View File

@@ -1,6 +1,6 @@
# [DEF:ConfigManagerModule:Module]
#
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: config, manager, persistence, migration, postgresql
# @PURPOSE: Manages application configuration persistence in DB with one-time migration from legacy JSON.
# @LAYER: Domain
@@ -29,7 +29,7 @@ from .logger import logger, configure_logger, belief_scope
# [DEF:ConfigManager:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Handles application configuration load, validation, mutation, and persistence lifecycle.
class ConfigManager:
# [DEF:__init__:Function]
@@ -60,7 +60,7 @@ class ConfigManager:
# [/DEF:__init__:Function]
# [DEF:_default_config:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Build default application configuration fallback.
# @PRE: None.
# @POST: Returns valid AppConfig with empty environments and default storage settings.
@@ -75,7 +75,7 @@ class ConfigManager:
# [/DEF:_default_config:Function]
# [DEF:_sync_raw_payload_from_config:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.
# @PRE: self.config is initialized as AppConfig.
# @POST: self.raw_payload contains AppConfig fields refreshed from self.config.
@@ -90,7 +90,7 @@ class ConfigManager:
# [/DEF:_sync_raw_payload_from_config:Function]
# [DEF:_load_from_legacy_file:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Load legacy JSON configuration for migration fallback path.
# @PRE: self.config_path is initialized.
# @POST: Returns AppConfig from file payload or safe default.
@@ -116,7 +116,7 @@ class ConfigManager:
# [/DEF:_load_from_legacy_file:Function]
# [DEF:_get_record:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Resolve global configuration record from DB.
# @PRE: session is an active SQLAlchemy Session.
# @POST: Returns record when present, otherwise None.
@@ -128,7 +128,7 @@ class ConfigManager:
# [/DEF:_get_record:Function]
# [DEF:_load_config:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON.
# @PRE: SessionLocal factory is available and AppConfigRecord schema is accessible.
# @POST: Returns valid AppConfig and closes opened DB session.
@@ -160,7 +160,7 @@ class ConfigManager:
# [/DEF:_load_config:Function]
# [DEF:_save_config_to_db:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Persist provided AppConfig into the global DB configuration record.
# @PRE: config is AppConfig; session is either None or an active Session.
# @POST: Global DB record payload equals config.model_dump() when commit succeeds.
@@ -195,7 +195,7 @@ class ConfigManager:
# [/DEF:_save_config_to_db:Function]
# [DEF:save:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Persist current in-memory configuration state.
# @PRE: self.config is initialized.
# @POST: Current self.config is written to DB global record.
@@ -207,7 +207,7 @@ class ConfigManager:
# [/DEF:save:Function]
# [DEF:get_config:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Return current in-memory configuration snapshot.
# @PRE: self.config is initialized.
# @POST: Returns AppConfig reference stored in manager.
@@ -219,7 +219,7 @@ class ConfigManager:
# [/DEF:get_config:Function]
# [DEF:get_payload:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema.
# @PRE: Manager state is initialized.
# @POST: Returns dict payload with current AppConfig fields synchronized.
@@ -231,7 +231,7 @@ class ConfigManager:
# [/DEF:get_payload:Function]
# [DEF:save_config:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict.
# @PRE: config is AppConfig or dict compatible with AppConfig core schema.
# @POST: self.config and self.raw_payload are synchronized and persisted to DB.
@@ -253,7 +253,7 @@ class ConfigManager:
# [/DEF:save_config:Function]
# [DEF:update_global_settings:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Replace global settings and persist the resulting configuration.
# @PRE: settings is GlobalSettings.
# @POST: self.config.settings equals provided settings and DB state is updated.
@@ -272,7 +272,7 @@ class ConfigManager:
# [/DEF:update_global_settings:Function]
# [DEF:validate_path:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Validate that path exists and is writable, creating it when absent.
# @PRE: path is a string path candidate.
# @POST: Returns (True, msg) for writable path, else (False, reason).
@@ -294,7 +294,7 @@ class ConfigManager:
# [/DEF:validate_path:Function]
# [DEF:get_environments:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Return all configured environments.
# @PRE: self.config is initialized.
# @POST: Returns list of Environment models from current configuration.
@@ -306,7 +306,7 @@ class ConfigManager:
# [/DEF:get_environments:Function]
# [DEF:has_environments:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Check whether at least one environment exists in configuration.
# @PRE: self.config is initialized.
# @POST: Returns True iff environment list length is greater than zero.
@@ -318,7 +318,7 @@ class ConfigManager:
# [/DEF:has_environments:Function]
# [DEF:get_environment:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Resolve a configured environment by identifier.
# @PRE: env_id is string identifier.
# @POST: Returns matching Environment when found; otherwise None.
@@ -333,7 +333,7 @@ class ConfigManager:
# [/DEF:get_environment:Function]
# [DEF:add_environment:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Upsert environment by id into configuration and persist.
# @PRE: env is Environment.
# @POST: Configuration contains provided env id with new payload persisted.
@@ -352,7 +352,7 @@ class ConfigManager:
# [/DEF:add_environment:Function]
# [DEF:update_environment:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior.
# @PRE: env_id is non-empty string and updated_env is Environment.
# @POST: Returns True and persists update when target exists; else returns False.
@@ -382,7 +382,7 @@ class ConfigManager:
# [/DEF:update_environment:Function]
# [DEF:delete_environment:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Delete environment by id and persist when deletion occurs.
# @PRE: env_id is non-empty string.
# @POST: Environment is removed when present; otherwise configuration is unchanged.

186
backend/src/core/config_models.py
View File

@@ -1,93 +1,93 @@
# [DEF:ConfigModels:Module]
# @TIER: STANDARD
# @SEMANTICS: config, models, pydantic
# @PURPOSE: Defines the data models for application configuration using Pydantic.
# @LAYER: Core
# @RELATION: READS_FROM -> app_configurations (database)
# @RELATION: USED_BY -> ConfigManager
from pydantic import BaseModel, Field
from typing import List, Optional
from ..models.storage import StorageConfig
from ..services.llm_prompt_templates import (
DEFAULT_LLM_ASSISTANT_SETTINGS,
DEFAULT_LLM_PROMPTS,
DEFAULT_LLM_PROVIDER_BINDINGS,
)
# [DEF:Schedule:DataClass]
# @PURPOSE: Represents a backup schedule configuration.
class Schedule(BaseModel):
enabled: bool = False
cron_expression: str = "0 0 * * *" # Default: daily at midnight
# [/DEF:Schedule:DataClass]
# [DEF:Environment:DataClass]
# @PURPOSE: Represents a Superset environment configuration.
class Environment(BaseModel):
id: str
name: str
url: str
username: str
password: str # Will be masked in UI
stage: str = Field(default="DEV", pattern="^(DEV|PREPROD|PROD)$")
verify_ssl: bool = True
timeout: int = 30
is_default: bool = False
is_production: bool = False
backup_schedule: Schedule = Field(default_factory=Schedule)
# [/DEF:Environment:DataClass]
# [DEF:LoggingConfig:DataClass]
# @PURPOSE: Defines the configuration for the application's logging system.
class LoggingConfig(BaseModel):
level: str = "INFO"
task_log_level: str = "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR)
file_path: Optional[str] = None
max_bytes: int = 10 * 1024 * 1024
backup_count: int = 5
enable_belief_state: bool = True
# [/DEF:LoggingConfig:DataClass]
# [DEF:CleanReleaseConfig:DataClass]
# @PURPOSE: Configuration for clean release compliance subsystem.
class CleanReleaseConfig(BaseModel):
active_policy_id: Optional[str] = None
active_registry_id: Optional[str] = None
# [/DEF:CleanReleaseConfig:DataClass]
# [DEF:GlobalSettings:DataClass]
# @PURPOSE: Represents global application settings.
class GlobalSettings(BaseModel):
storage: StorageConfig = Field(default_factory=StorageConfig)
clean_release: CleanReleaseConfig = Field(default_factory=CleanReleaseConfig)
default_environment_id: Optional[str] = None
logging: LoggingConfig = Field(default_factory=LoggingConfig)
connections: List[dict] = []
llm: dict = Field(
default_factory=lambda: {
"providers": [],
"default_provider": "",
"prompts": dict(DEFAULT_LLM_PROMPTS),
"provider_bindings": dict(DEFAULT_LLM_PROVIDER_BINDINGS),
**dict(DEFAULT_LLM_ASSISTANT_SETTINGS),
}
)
# Task retention settings
task_retention_days: int = 30
task_retention_limit: int = 100
pagination_limit: int = 10
# Migration sync settings
migration_sync_cron: str = "0 2 * * *"
# [/DEF:GlobalSettings:DataClass]
# [DEF:AppConfig:DataClass]
# @PURPOSE: The root configuration model containing all application settings.
class AppConfig(BaseModel):
environments: List[Environment] = []
settings: GlobalSettings
# [/DEF:AppConfig:DataClass]
# [/DEF:ConfigModels:Module]
# [DEF:backend.src.core.config_models:Module]
# @COMPLEXITY: 3
# @SEMANTICS: config, models, pydantic
# @PURPOSE: Defines the data models for application configuration using Pydantic.
# @LAYER: Core
# @RELATION: READS_FROM -> app_configurations (database)
# @RELATION: USED_BY -> ConfigManager
from pydantic import BaseModel, Field
from typing import List, Optional
from ..models.storage import StorageConfig
from ..services.llm_prompt_templates import (
DEFAULT_LLM_ASSISTANT_SETTINGS,
DEFAULT_LLM_PROMPTS,
DEFAULT_LLM_PROVIDER_BINDINGS,
)
# [DEF:Schedule:DataClass]
# @PURPOSE: Represents a backup schedule configuration.
class Schedule(BaseModel):
enabled: bool = False
cron_expression: str = "0 0 * * *" # Default: daily at midnight
# [/DEF:Schedule:DataClass]
# [DEF:backend.src.core.config_models.Environment:DataClass]
# @PURPOSE: Represents a Superset environment configuration.
class Environment(BaseModel):
id: str
name: str
url: str
username: str
password: str # Will be masked in UI
stage: str = Field(default="DEV", pattern="^(DEV|PREPROD|PROD)$")
verify_ssl: bool = True
timeout: int = 30
is_default: bool = False
is_production: bool = False
backup_schedule: Schedule = Field(default_factory=Schedule)
# [/DEF:backend.src.core.config_models.Environment:DataClass]
# [DEF:LoggingConfig:DataClass]
# @PURPOSE: Defines the configuration for the application's logging system.
class LoggingConfig(BaseModel):
level: str = "INFO"
task_log_level: str = "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR)
file_path: Optional[str] = None
max_bytes: int = 10 * 1024 * 1024
backup_count: int = 5
enable_belief_state: bool = True
# [/DEF:LoggingConfig:DataClass]
# [DEF:CleanReleaseConfig:DataClass]
# @PURPOSE: Configuration for clean release compliance subsystem.
class CleanReleaseConfig(BaseModel):
active_policy_id: Optional[str] = None
active_registry_id: Optional[str] = None
# [/DEF:CleanReleaseConfig:DataClass]
# [DEF:GlobalSettings:DataClass]
# @PURPOSE: Represents global application settings.
class GlobalSettings(BaseModel):
storage: StorageConfig = Field(default_factory=StorageConfig)
clean_release: CleanReleaseConfig = Field(default_factory=CleanReleaseConfig)
default_environment_id: Optional[str] = None
logging: LoggingConfig = Field(default_factory=LoggingConfig)
connections: List[dict] = []
llm: dict = Field(
default_factory=lambda: {
"providers": [],
"default_provider": "",
"prompts": dict(DEFAULT_LLM_PROMPTS),
"provider_bindings": dict(DEFAULT_LLM_PROVIDER_BINDINGS),
**dict(DEFAULT_LLM_ASSISTANT_SETTINGS),
}
)
# Task retention settings
task_retention_days: int = 30
task_retention_limit: int = 100
pagination_limit: int = 10
# Migration sync settings
migration_sync_cron: str = "0 2 * * *"
# [/DEF:GlobalSettings:DataClass]
# [DEF:AppConfig:DataClass]
# @PURPOSE: The root configuration model containing all application settings.
class AppConfig(BaseModel):
environments: List[Environment] = []
settings: GlobalSettings
# [/DEF:AppConfig:DataClass]
# [/DEF:ConfigModels:Module]

40
backend/src/core/database.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.database:Module]
#
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: database, postgresql, sqlalchemy, session, persistence
# @PURPOSE: Configures database connection and session management (PostgreSQL-first).
# @LAYER: Core
@@ -31,13 +31,13 @@ from pathlib import Path
# [/SECTION]
# [DEF:BASE_DIR:Variable]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Base directory for the backend.
BASE_DIR = Path(__file__).resolve().parent.parent.parent
# [/DEF:BASE_DIR:Variable]
# [DEF:DATABASE_URL:Constant]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: URL for the main application database.
DEFAULT_POSTGRES_URL = os.getenv(
"POSTGRES_URL",
@@ -47,20 +47,20 @@ DATABASE_URL = os.getenv("DATABASE_URL", DEFAULT_POSTGRES_URL)
# [/DEF:DATABASE_URL:Constant]
# [DEF:TASKS_DATABASE_URL:Constant]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: URL for the tasks execution database.
# Defaults to DATABASE_URL to keep task logs in the same PostgreSQL instance.
TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", DATABASE_URL)
# [/DEF:TASKS_DATABASE_URL:Constant]
# [DEF:AUTH_DATABASE_URL:Constant]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: URL for the authentication database.
AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL)
# [/DEF:AUTH_DATABASE_URL:Constant]
# [DEF:engine:Variable]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: SQLAlchemy engine for mappings database.
# @SIDE_EFFECT: Creates database engine and manages connection pool.
def _build_engine(db_url: str):
@@ -73,40 +73,40 @@ engine = _build_engine(DATABASE_URL)
# [/DEF:engine:Variable]
# [DEF:tasks_engine:Variable]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: SQLAlchemy engine for tasks database.
tasks_engine = _build_engine(TASKS_DATABASE_URL)
# [/DEF:tasks_engine:Variable]
# [DEF:auth_engine:Variable]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: SQLAlchemy engine for authentication database.
auth_engine = _build_engine(AUTH_DATABASE_URL)
# [/DEF:auth_engine:Variable]
# [DEF:SessionLocal:Class]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: A session factory for the main mappings database.
# @PRE: engine is initialized.
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# [/DEF:SessionLocal:Class]
# [DEF:TasksSessionLocal:Class]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: A session factory for the tasks execution database.
# @PRE: tasks_engine is initialized.
TasksSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=tasks_engine)
# [/DEF:TasksSessionLocal:Class]
# [DEF:AuthSessionLocal:Class]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: A session factory for the authentication database.
# @PRE: auth_engine is initialized.
AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_engine)
# [/DEF:AuthSessionLocal:Class]
# [DEF:_ensure_user_dashboard_preferences_columns:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table.
# @PRE: bind_engine points to application database where profile table is stored.
# @POST: Missing columns are added without data loss.
@@ -173,7 +173,7 @@ def _ensure_user_dashboard_preferences_columns(bind_engine):
# [DEF:_ensure_user_dashboard_preferences_health_columns:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table (health fields).
def _ensure_user_dashboard_preferences_health_columns(bind_engine):
with belief_scope("_ensure_user_dashboard_preferences_health_columns"):
@@ -217,7 +217,7 @@ def _ensure_user_dashboard_preferences_health_columns(bind_engine):
# [DEF:_ensure_llm_validation_results_columns:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for llm_validation_results table.
def _ensure_llm_validation_results_columns(bind_engine):
with belief_scope("_ensure_llm_validation_results_columns"):
@@ -257,7 +257,7 @@ def _ensure_llm_validation_results_columns(bind_engine):
# [DEF:_ensure_git_server_configs_columns:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Applies additive schema upgrades for git_server_configs table.
# @PRE: bind_engine points to application database.
# @POST: Missing columns are added without data loss.
@@ -295,7 +295,7 @@ def _ensure_git_server_configs_columns(bind_engine):
# [DEF:ensure_connection_configs_table:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Ensures the external connection registry table exists in the main database.
# @PRE: bind_engine points to the application database.
# @POST: connection_configs table exists without dropping existing data.
@@ -313,7 +313,7 @@ def ensure_connection_configs_table(bind_engine):
# [DEF:init_db:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Initializes the database by creating all tables.
# @PRE: engine, tasks_engine and auth_engine are initialized.
# @POST: Database tables created in all databases.
@@ -331,7 +331,7 @@ def init_db():
# [/DEF:init_db:Function]
# [DEF:get_db:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Dependency for getting a database session.
# @PRE: SessionLocal is initialized.
# @POST: Session is closed after use.
@@ -346,7 +346,7 @@ def get_db():
# [/DEF:get_db:Function]
# [DEF:get_tasks_db:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Dependency for getting a tasks database session.
# @PRE: TasksSessionLocal is initialized.
# @POST: Session is closed after use.
@@ -361,7 +361,7 @@ def get_tasks_db():
# [/DEF:get_tasks_db:Function]
# [DEF:get_auth_db:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Dependency for getting an authentication database session.
# @PRE: AuthSessionLocal is initialized.
# @POST: Session is closed after use.

2
backend/src/core/encryption_key.py
View File

@@ -1,5 +1,5 @@
# [DEF:backend.src.core.encryption_key:Module]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: encryption, key, bootstrap, environment, startup
# @PURPOSE: Resolve and persist the Fernet encryption key required by runtime services.
# @LAYER: Infra

2
backend/src/core/logger/__tests__/test_logger.py
View File

@@ -1,5 +1,5 @@
# [DEF:test_logger:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for logger module
# @LAYER: Infra
# @RELATION: VERIFIES -> src.core.logger

4
backend/src/core/mapping_service.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.mapping_service:Module]
#
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: mapping, ids, synchronization, environments, cross-filters
# @PURPOSE: Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)
# @LAYER: Core
@@ -21,7 +21,7 @@ from src.core.logger import logger, belief_scope
# [/SECTION]
# [DEF:IdMappingService:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs.
#
# @TEST_CONTRACT: IdMappingServiceModel ->

2
backend/src/core/migration/__init__.py
View File

@@ -1,5 +1,5 @@
# [DEF:backend.src.core.migration.__init__:Module]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @SEMANTICS: migration, package, exports
# @PURPOSE: Namespace package for migration pre-flight orchestration components.
# @LAYER: Core

2
backend/src/core/migration/archive_parser.py
View File

@@ -1,5 +1,5 @@
# [DEF:backend.src.core.migration.archive_parser:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: migration, zip, parser, yaml, metadata
# @PURPOSE: Parse Superset export ZIP archives into normalized object catalogs for diffing.
# @LAYER: Core

10
backend/src/core/migration/dry_run_orchestrator.py
View File

@@ -1,12 +1,12 @@
# [DEF:backend.src.core.migration.dry_run_orchestrator:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: migration, dry_run, diff, risk, superset
# @PURPOSE: Compute pre-flight migration diff and risk scoring without apply.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> backend.src.core.superset_client
# @RELATION: DEPENDS_ON -> backend.src.core.migration_engine
# @RELATION: DEPENDS_ON -> backend.src.core.migration.archive_parser
# @RELATION: DEPENDS_ON -> backend.src.core.migration.risk_assessor
# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration_engine.MigrationEngine]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration.archive_parser.MigrationArchiveParser]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration.risk_assessor]
# @INVARIANT: Dry run is informative only and must not mutate target environment.
from datetime import datetime, timezone

2
backend/src/core/migration/risk_assessor.py
View File

@@ -1,5 +1,5 @@
# [DEF:backend.src.core.migration.risk_assessor:Module]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: migration, dry_run, risk, scoring, preflight
# @PURPOSE: Compute deterministic migration risk items and aggregate score for dry-run reporting.
# @LAYER: Domain

2
backend/src/core/migration_engine.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.migration_engine:Module]
#
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: migration, engine, zip, yaml, transformation, cross-filter, id-mapping
# @PURPOSE: Transforms Superset export ZIP archives while preserving archive integrity and patching mapped identifiers.
# @LAYER: Domain

378
backend/src/core/plugin_loader.py
View File

@@ -1,192 +1,192 @@
import importlib.util
import os
import sys # Added this line
from typing import Dict, List, Optional
from .plugin_base import PluginBase, PluginConfig
from .logger import belief_scope
# [DEF:PluginLoader:Class]
# @TIER: STANDARD
# @SEMANTICS: plugin, loader, dynamic, import
# @PURPOSE: Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
# @LAYER: Core
# @RELATION: Depends on PluginBase. It is used by the main application to discover and manage available plugins.
class PluginLoader:
"""
Scans a directory for Python modules, loads them, and identifies classes
that inherit from PluginBase.
"""
# [DEF:__init__:Function]
# @PURPOSE: Initializes the PluginLoader with a directory to scan.
# @PRE: plugin_dir is a valid directory path.
# @POST: Plugins are loaded and registered.
# @PARAM: plugin_dir (str) - The directory containing plugin modules.
def __init__(self, plugin_dir: str):
with belief_scope("__init__"):
self.plugin_dir = plugin_dir
self._plugins: Dict[str, PluginBase] = {}
self._plugin_configs: Dict[str, PluginConfig] = {}
self._load_plugins()
# [/DEF:__init__:Function]
# [DEF:_load_plugins:Function]
# @PURPOSE: Scans the plugin directory and loads all valid plugins.
# @PRE: plugin_dir exists or can be created.
# @POST: _load_module is called for each .py file.
def _load_plugins(self):
with belief_scope("_load_plugins"):
"""
Scans the plugin directory, imports modules, and registers valid plugins.
"""
if not os.path.exists(self.plugin_dir):
os.makedirs(self.plugin_dir)
# Add the plugin directory's parent to sys.path to enable relative imports within plugins
# This assumes plugin_dir is something like 'backend/src/plugins'
# and we want 'backend/src' to be on the path for 'from ..core...' imports
plugin_parent_dir = os.path.abspath(os.path.join(self.plugin_dir, os.pardir))
if plugin_parent_dir not in sys.path:
sys.path.insert(0, plugin_parent_dir)
for filename in os.listdir(self.plugin_dir):
file_path = os.path.join(self.plugin_dir, filename)
# Handle directory-based plugins (packages)
if os.path.isdir(file_path):
init_file = os.path.join(file_path, "__init__.py")
if os.path.exists(init_file):
self._load_module(filename, init_file)
continue
# Handle single-file plugins
if filename.endswith(".py") and filename != "__init__.py":
module_name = filename[:-3]
self._load_module(module_name, file_path)
# [/DEF:_load_plugins:Function]
# [DEF:_load_module:Function]
# @PURPOSE: Loads a single Python module and discovers PluginBase implementations.
# @PRE: module_name and file_path are valid.
# @POST: Plugin classes are instantiated and registered.
# @PARAM: module_name (str) - The name of the module.
# @PARAM: file_path (str) - The path to the module file.
def _load_module(self, module_name: str, file_path: str):
with belief_scope("_load_module"):
"""
Loads a single Python module and extracts PluginBase subclasses.
"""
import importlib.util
import os
import sys # Added this line
from typing import Dict, List, Optional
from .plugin_base import PluginBase, PluginConfig
from .logger import belief_scope
# [DEF:PluginLoader:Class]
# @COMPLEXITY: 3
# @SEMANTICS: plugin, loader, dynamic, import
# @PURPOSE: Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
# @LAYER: Core
# @RELATION: Depends on PluginBase. It is used by the main application to discover and manage available plugins.
class PluginLoader:
"""
Scans a directory for Python modules, loads them, and identifies classes
that inherit from PluginBase.
"""
# [DEF:__init__:Function]
# @PURPOSE: Initializes the PluginLoader with a directory to scan.
# @PRE: plugin_dir is a valid directory path.
# @POST: Plugins are loaded and registered.
# @PARAM: plugin_dir (str) - The directory containing plugin modules.
def __init__(self, plugin_dir: str):
with belief_scope("__init__"):
self.plugin_dir = plugin_dir
self._plugins: Dict[str, PluginBase] = {}
self._plugin_configs: Dict[str, PluginConfig] = {}
self._load_plugins()
# [/DEF:__init__:Function]
# [DEF:_load_plugins:Function]
# @PURPOSE: Scans the plugin directory and loads all valid plugins.
# @PRE: plugin_dir exists or can be created.
# @POST: _load_module is called for each .py file.
def _load_plugins(self):
with belief_scope("_load_plugins"):
"""
Scans the plugin directory, imports modules, and registers valid plugins.
"""
if not os.path.exists(self.plugin_dir):
os.makedirs(self.plugin_dir)
# Add the plugin directory's parent to sys.path to enable relative imports within plugins
# This assumes plugin_dir is something like 'backend/src/plugins'
# and we want 'backend/src' to be on the path for 'from ..core...' imports
plugin_parent_dir = os.path.abspath(os.path.join(self.plugin_dir, os.pardir))
if plugin_parent_dir not in sys.path:
sys.path.insert(0, plugin_parent_dir)
for filename in os.listdir(self.plugin_dir):
file_path = os.path.join(self.plugin_dir, filename)
# Handle directory-based plugins (packages)
if os.path.isdir(file_path):
init_file = os.path.join(file_path, "__init__.py")
if os.path.exists(init_file):
self._load_module(filename, init_file)
continue
# Handle single-file plugins
if filename.endswith(".py") and filename != "__init__.py":
module_name = filename[:-3]
self._load_module(module_name, file_path)
# [/DEF:_load_plugins:Function]
# [DEF:_load_module:Function]
# @PURPOSE: Loads a single Python module and discovers PluginBase implementations.
# @PRE: module_name and file_path are valid.
# @POST: Plugin classes are instantiated and registered.
# @PARAM: module_name (str) - The name of the module.
# @PARAM: file_path (str) - The path to the module file.
def _load_module(self, module_name: str, file_path: str):
with belief_scope("_load_module"):
"""
Loads a single Python module and extracts PluginBase subclasses.
"""
# All runtime code is imported through the canonical `src` package root.
package_name = f"src.plugins.{module_name}"
# print(f"DEBUG: Loading plugin {module_name} as {package_name}")
spec = importlib.util.spec_from_file_location(package_name, file_path)
if spec is None or spec.loader is None:
print(f"Could not load module spec for {package_name}") # Replace with proper logging
return
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module)
except Exception as e:
print(f"Error loading plugin module {module_name}: {e}") # Replace with proper logging
return
for attribute_name in dir(module):
attribute = getattr(module, attribute_name)
if (
isinstance(attribute, type)
and issubclass(attribute, PluginBase)
and attribute is not PluginBase
):
try:
plugin_instance = attribute()
self._register_plugin(plugin_instance)
except Exception as e:
print(f"Error instantiating plugin {attribute_name} in {module_name}: {e}") # Replace with proper logging
# [/DEF:_load_module:Function]
# [DEF:_register_plugin:Function]
# @PURPOSE: Registers a PluginBase instance and its configuration.
# @PRE: plugin_instance is a valid implementation of PluginBase.
# @POST: Plugin is added to _plugins and _plugin_configs.
# @PARAM: plugin_instance (PluginBase) - The plugin instance to register.
def _register_plugin(self, plugin_instance: PluginBase):
with belief_scope("_register_plugin"):
"""
Registers a valid plugin instance.
"""
plugin_id = plugin_instance.id
if plugin_id in self._plugins:
print(f"Warning: Duplicate plugin ID '{plugin_id}' found. Skipping.") # Replace with proper logging
return
try:
schema = plugin_instance.get_schema()
# Basic validation to ensure it's a dictionary
if not isinstance(schema, dict):
raise TypeError("get_schema() must return a dictionary.")
plugin_config = PluginConfig(
id=plugin_instance.id,
name=plugin_instance.name,
description=plugin_instance.description,
version=plugin_instance.version,
ui_route=plugin_instance.ui_route,
schema=schema,
)
# The following line is commented out because it requires a schema to be passed to validate against.
# The schema provided by the plugin is the one being validated, not the data.
# validate(instance={}, schema=schema)
self._plugins[plugin_id] = plugin_instance
self._plugin_configs[plugin_id] = plugin_config
from ..core.logger import logger
logger.info(f"Plugin '{plugin_instance.name}' (ID: {plugin_id}) loaded successfully.")
except Exception as e:
from ..core.logger import logger
logger.error(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}")
# [/DEF:_register_plugin:Function]
# [DEF:get_plugin:Function]
# @PURPOSE: Retrieves a loaded plugin instance by its ID.
# @PRE: plugin_id is a string.
# @POST: Returns plugin instance or None.
# @PARAM: plugin_id (str) - The unique identifier of the plugin.
# @RETURN: Optional[PluginBase] - The plugin instance if found, otherwise None.
def get_plugin(self, plugin_id: str) -> Optional[PluginBase]:
with belief_scope("get_plugin"):
"""
Returns a loaded plugin instance by its ID.
"""
return self._plugins.get(plugin_id)
# [/DEF:get_plugin:Function]
# [DEF:get_all_plugin_configs:Function]
# @PURPOSE: Returns a list of all registered plugin configurations.
# @PRE: None.
# @POST: Returns list of all PluginConfig objects.
# @RETURN: List[PluginConfig] - A list of plugin configurations.
def get_all_plugin_configs(self) -> List[PluginConfig]:
with belief_scope("get_all_plugin_configs"):
"""
Returns a list of all loaded plugin configurations.
"""
return list(self._plugin_configs.values())
# [/DEF:get_all_plugin_configs:Function]
# [DEF:has_plugin:Function]
# @PURPOSE: Checks if a plugin with the given ID is registered.
# @PRE: plugin_id is a string.
# @POST: Returns True if plugin exists.
# @PARAM: plugin_id (str) - The unique identifier of the plugin.
# @RETURN: bool - True if the plugin is registered, False otherwise.
def has_plugin(self, plugin_id: str) -> bool:
with belief_scope("has_plugin"):
"""
Checks if a plugin with the given ID is loaded.
"""
return plugin_id in self._plugins
# [/DEF:has_plugin:Function]
# print(f"DEBUG: Loading plugin {module_name} as {package_name}")
spec = importlib.util.spec_from_file_location(package_name, file_path)
if spec is None or spec.loader is None:
print(f"Could not load module spec for {package_name}") # Replace with proper logging
return
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module)
except Exception as e:
print(f"Error loading plugin module {module_name}: {e}") # Replace with proper logging
return
for attribute_name in dir(module):
attribute = getattr(module, attribute_name)
if (
isinstance(attribute, type)
and issubclass(attribute, PluginBase)
and attribute is not PluginBase
):
try:
plugin_instance = attribute()
self._register_plugin(plugin_instance)
except Exception as e:
print(f"Error instantiating plugin {attribute_name} in {module_name}: {e}") # Replace with proper logging
# [/DEF:_load_module:Function]
# [DEF:_register_plugin:Function]
# @PURPOSE: Registers a PluginBase instance and its configuration.
# @PRE: plugin_instance is a valid implementation of PluginBase.
# @POST: Plugin is added to _plugins and _plugin_configs.
# @PARAM: plugin_instance (PluginBase) - The plugin instance to register.
def _register_plugin(self, plugin_instance: PluginBase):
with belief_scope("_register_plugin"):
"""
Registers a valid plugin instance.
"""
plugin_id = plugin_instance.id
if plugin_id in self._plugins:
print(f"Warning: Duplicate plugin ID '{plugin_id}' found. Skipping.") # Replace with proper logging
return
try:
schema = plugin_instance.get_schema()
# Basic validation to ensure it's a dictionary
if not isinstance(schema, dict):
raise TypeError("get_schema() must return a dictionary.")
plugin_config = PluginConfig(
id=plugin_instance.id,
name=plugin_instance.name,
description=plugin_instance.description,
version=plugin_instance.version,
ui_route=plugin_instance.ui_route,
schema=schema,
)
# The following line is commented out because it requires a schema to be passed to validate against.
# The schema provided by the plugin is the one being validated, not the data.
# validate(instance={}, schema=schema)
self._plugins[plugin_id] = plugin_instance
self._plugin_configs[plugin_id] = plugin_config
from ..core.logger import logger
logger.info(f"Plugin '{plugin_instance.name}' (ID: {plugin_id}) loaded successfully.")
except Exception as e:
from ..core.logger import logger
logger.error(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}")
# [/DEF:_register_plugin:Function]
# [DEF:get_plugin:Function]
# @PURPOSE: Retrieves a loaded plugin instance by its ID.
# @PRE: plugin_id is a string.
# @POST: Returns plugin instance or None.
# @PARAM: plugin_id (str) - The unique identifier of the plugin.
# @RETURN: Optional[PluginBase] - The plugin instance if found, otherwise None.
def get_plugin(self, plugin_id: str) -> Optional[PluginBase]:
with belief_scope("get_plugin"):
"""
Returns a loaded plugin instance by its ID.
"""
return self._plugins.get(plugin_id)
# [/DEF:get_plugin:Function]
# [DEF:get_all_plugin_configs:Function]
# @PURPOSE: Returns a list of all registered plugin configurations.
# @PRE: None.
# @POST: Returns list of all PluginConfig objects.
# @RETURN: List[PluginConfig] - A list of plugin configurations.
def get_all_plugin_configs(self) -> List[PluginConfig]:
with belief_scope("get_all_plugin_configs"):
"""
Returns a list of all loaded plugin configurations.
"""
return list(self._plugin_configs.values())
# [/DEF:get_all_plugin_configs:Function]
# [DEF:has_plugin:Function]
# @PURPOSE: Checks if a plugin with the given ID is registered.
# @PRE: plugin_id is a string.
# @POST: Returns True if plugin exists.
# @PARAM: plugin_id (str) - The unique identifier of the plugin.
# @RETURN: bool - True if the plugin is registered, False otherwise.
def has_plugin(self, plugin_id: str) -> bool:
with belief_scope("has_plugin"):
"""
Checks if a plugin with the given ID is loaded.
"""
return plugin_id in self._plugins
# [/DEF:has_plugin:Function]
# [/DEF:PluginLoader:Class]

6
backend/src/core/scheduler.py
View File

@@ -1,5 +1,5 @@
# [DEF:SchedulerModule:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: scheduler, apscheduler, cron, backup
# @PURPOSE: Manages scheduled tasks using APScheduler.
# @LAYER: Core
@@ -18,7 +18,7 @@ from datetime import datetime, time, timedelta, date
# [/SECTION]
# [DEF:SchedulerService:Class]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: scheduler, service, apscheduler
# @PURPOSE: Provides a service to manage scheduled backup tasks.
class SchedulerService:
@@ -123,7 +123,7 @@ class SchedulerService:
# [/DEF:SchedulerService:Class]
# [DEF:ThrottledSchedulerConfigurator:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: scheduler, throttling, distribution
# @PURPOSE: Distributes validation tasks evenly within an execution window.
class ThrottledSchedulerConfigurator:

76
backend/src/core/superset_client.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.superset_client:Module]
#
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: superset, api, client, rest, http, dashboard, dataset, import, export
# @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию.
# @LAYER: Core
@@ -25,13 +25,13 @@ from .config_models import Environment
# [/SECTION]
# [DEF:backend.src.core.superset_client.SupersetClient:Class]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Класс-обёртка над Superset REST API, предоставляющий методы для работы с дашбордами и датасетами.
# @RELATION: [DEPENDS_ON] ->[backend.src.core.utils.network.APIClient]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.config_models.Environment]
class SupersetClient:
# [DEF:backend.src.core.superset_client.SupersetClient.__init__:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
# @PRE: `env` должен быть валидным объектом Environment.
# @POST: Атрибуты `env` и `network` созданы и готовы к работе.
@@ -60,7 +60,7 @@ class SupersetClient:
# [/DEF:__init__:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.authenticate:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Authenticates the client using the configured credentials.
# @PRE: self.network must be initialized with valid auth configuration.
# @POST: Client is authenticated and tokens are stored.
@@ -73,7 +73,7 @@ class SupersetClient:
@property
# [DEF:backend.src.core.superset_client.SupersetClient.headers:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
# @PRE: APIClient is initialized and authenticated.
# @POST: Returns a dictionary of HTTP headers.
@@ -85,7 +85,7 @@ class SupersetClient:
# [SECTION: DASHBOARD OPERATIONS]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.
# @PRE: Client is authenticated.
# @POST: Returns a tuple with total count and list of dashboards.
@@ -119,7 +119,7 @@ class SupersetClient:
# [/DEF:get_dashboards:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_page:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.
# @PRE: Client is authenticated.
# @POST: Returns total count and one page of dashboards.
@@ -156,7 +156,7 @@ class SupersetClient:
# [/DEF:get_dashboards_page:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_summary:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches dashboard metadata optimized for the grid.
# @PRE: Client is authenticated.
# @POST: Returns a list of dashboard metadata summaries.
@@ -241,7 +241,7 @@ class SupersetClient:
# [/DEF:get_dashboards_summary:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_summary_page:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.
# @PRE: page >= 1 and page_size > 0.
# @POST: Returns mapped summaries and total dashboard count.
@@ -314,7 +314,7 @@ class SupersetClient:
# [/DEF:get_dashboards_summary_page:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._extract_owner_labels:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Normalize dashboard owners payload to stable display labels.
# @PRE: owners payload can be scalar, object or list.
# @POST: Returns deduplicated non-empty owner labels preserving order.
@@ -342,7 +342,7 @@ class SupersetClient:
# [/DEF:_extract_owner_labels:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._extract_user_display:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Normalize user payload to a stable display name.
# @PRE: user payload can be string, dict or None.
# @POST: Returns compact non-empty display value or None.
@@ -371,7 +371,7 @@ class SupersetClient:
# [/DEF:_extract_user_display:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._sanitize_user_text:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Convert scalar value to non-empty user-facing text.
# @PRE: value can be any scalar type.
# @POST: Returns trimmed string or None.
@@ -385,7 +385,7 @@ class SupersetClient:
# [/DEF:_sanitize_user_text:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboard:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single dashboard by ID.
# @PRE: Client is authenticated and dashboard_id exists.
# @POST: Returns dashboard payload from Superset API.
@@ -398,7 +398,7 @@ class SupersetClient:
# [/DEF:get_dashboard:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_chart:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single chart by ID.
# @PRE: Client is authenticated and chart_id exists.
# @POST: Returns chart payload from Superset API.
@@ -411,7 +411,7 @@ class SupersetClient:
# [/DEF:get_chart:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboard_detail:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches detailed dashboard information including related charts and datasets.
# @PRE: Client is authenticated and dashboard_id exists.
# @POST: Returns dashboard metadata with charts and datasets lists.
@@ -606,7 +606,7 @@ class SupersetClient:
# [/DEF:get_dashboard_detail:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_charts:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches all charts with pagination support.
# @PRE: Client is authenticated.
# @POST: Returns total count and charts list.
@@ -626,7 +626,7 @@ class SupersetClient:
# [/DEF:get_charts:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._extract_chart_ids_from_layout:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys.
# @PRE: payload can be dict/list/scalar.
# @POST: Returns a set of chart IDs found in nested structures.
@@ -659,7 +659,7 @@ class SupersetClient:
# [/DEF:_extract_chart_ids_from_layout:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.export_dashboard:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Экспортирует дашборд в виде ZIP-архива.
# @PRE: dashboard_id must exist in Superset.
# @POST: Returns ZIP content and filename.
@@ -684,7 +684,7 @@ class SupersetClient:
# [/DEF:export_dashboard:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.import_dashboard:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Импортирует дашборд из ZIP-файла.
# @PRE: file_name must be a valid ZIP dashboard export.
# @POST: Dashboard is imported or re-imported after deletion.
@@ -716,7 +716,7 @@ class SupersetClient:
# [/DEF:import_dashboard:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.delete_dashboard:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Удаляет дашборд по его ID или slug.
# @PRE: dashboard_id must exist.
# @POST: Dashboard is removed from Superset.
@@ -738,7 +738,7 @@ class SupersetClient:
# [SECTION: DATASET OPERATIONS]
# [DEF:backend.src.core.superset_client.SupersetClient.get_datasets:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.
# @PRE: Client is authenticated.
# @POST: Returns total count and list of datasets.
@@ -759,7 +759,7 @@ class SupersetClient:
# [/DEF:get_datasets:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_datasets_summary:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.
# @PRE: Client is authenticated.
# @POST: Returns a list of dataset metadata summaries.
@@ -784,7 +784,7 @@ class SupersetClient:
# [/DEF:get_datasets_summary:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dataset_detail:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches detailed dataset information including columns and linked dashboards
# @PRE: Client is authenticated and dataset_id exists.
# @POST: Returns detailed dataset info with columns and linked dashboards.
@@ -897,7 +897,7 @@ class SupersetClient:
# [/DEF:get_dataset_detail:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_dataset:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Получает информацию о конкретном датасете по его ID.
# @PRE: dataset_id must exist.
# @POST: Returns dataset details.
@@ -913,7 +913,7 @@ class SupersetClient:
# [/DEF:get_dataset:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.update_dataset:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Обновляет данные датасета по его ID.
# @PRE: dataset_id must exist.
# @POST: Dataset is updated in Superset.
@@ -939,7 +939,7 @@ class SupersetClient:
# [SECTION: DATABASE OPERATIONS]
# [DEF:backend.src.core.superset_client.SupersetClient.get_databases:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список баз данных.
# @PRE: Client is authenticated.
# @POST: Returns total count and list of databases.
@@ -962,7 +962,7 @@ class SupersetClient:
# [/DEF:get_databases:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_database:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Получает информацию о конкретной базе данных по её ID.
# @PRE: database_id must exist.
# @POST: Returns database details.
@@ -978,7 +978,7 @@ class SupersetClient:
# [/DEF:get_database:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_databases_summary:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetch a summary of databases including uuid, name, and engine.
# @PRE: Client is authenticated.
# @POST: Returns list of database summaries.
@@ -999,7 +999,7 @@ class SupersetClient:
# [/DEF:get_databases_summary:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_database_by_uuid:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Find a database by its UUID.
# @PRE: db_uuid must be a valid UUID string.
# @POST: Returns database info or None.
@@ -1019,7 +1019,7 @@ class SupersetClient:
# [SECTION: HELPERS]
# [DEF:backend.src.core.superset_client.SupersetClient._resolve_target_id_for_delete:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Resolves a dashboard ID from either an ID or a slug.
# @PRE: Either dash_id or dash_slug should be provided.
# @POST: Returns the resolved ID or None.
@@ -1042,7 +1042,7 @@ class SupersetClient:
# [/DEF:_resolve_target_id_for_delete:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._do_import:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Performs the actual multipart upload for import.
# @PRE: file_name must be a path to an existing ZIP file.
# @POST: Returns the API response from the upload.
@@ -1064,7 +1064,7 @@ class SupersetClient:
# [/DEF:_do_import:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._validate_export_response:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Validates that the export response is a non-empty ZIP archive.
# @PRE: response must be a valid requests.Response object.
# @POST: Raises SupersetAPIError if validation fails.
@@ -1078,7 +1078,7 @@ class SupersetClient:
# [/DEF:_validate_export_response:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._resolve_export_filename:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Determines the filename for an exported dashboard.
# @PRE: response must contain Content-Disposition header or dashboard_id must be provided.
# @POST: Returns a sanitized filename string.
@@ -1094,7 +1094,7 @@ class SupersetClient:
# [/DEF:_resolve_export_filename:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._validate_query_params:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Ensures query parameters have default page and page_size.
# @PRE: query can be None or a dictionary.
# @POST: Returns a dictionary with at least page and page_size.
@@ -1107,7 +1107,7 @@ class SupersetClient:
# [/DEF:_validate_query_params:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._fetch_total_object_count:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Fetches the total number of items for a given endpoint.
# @PRE: endpoint must be a valid Superset API path.
# @POST: Returns the total count as an integer.
@@ -1122,7 +1122,7 @@ class SupersetClient:
# [/DEF:_fetch_total_object_count:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._fetch_all_pages:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Iterates through all pages to collect all data items.
# @PRE: pagination_options must contain base_query, total_count, and results_field.
# @POST: Returns a combined list of all items.
@@ -1132,7 +1132,7 @@ class SupersetClient:
# [/DEF:_fetch_all_pages:Function]
# [DEF:backend.src.core.superset_client.SupersetClient._validate_import_file:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml.
# @PRE: zip_path must be a path to a file.
# @POST: Raises error if file is missing, not a ZIP, or missing metadata.
@@ -1149,7 +1149,7 @@ class SupersetClient:
# [/DEF:_validate_import_file:Function]
# [DEF:backend.src.core.superset_client.SupersetClient.get_all_resources:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.
# @PARAM: resource_type (str) - One of "chart", "dataset", "dashboard".
# @PRE: Client is authenticated. resource_type is valid.

4
backend/src/core/superset_profile_lookup.py
View File

@@ -1,6 +1,6 @@
# [DEF:backend.src.core.superset_profile_lookup:Module]
#
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: superset, users, lookup, profile, pagination, normalization
# @PURPOSE: Provides environment-scoped Superset account lookup adapter with stable normalized output.
# @LAYER: Core
@@ -19,7 +19,7 @@ from .utils.network import APIClient, AuthenticationError, SupersetAPIError
# [DEF:SupersetAccountLookupAdapter:Class]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Lookup Superset users and normalize candidates for profile binding.
class SupersetAccountLookupAdapter:
# [DEF:__init__:Function]

7
backend/src/core/task_manager/__init__.py
View File

@@ -1,9 +1,12 @@
# [DEF:TaskManagerPackage:Module]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @SEMANTICS: task, manager, package, exports
# @PURPOSE: Exports the public API of the task manager package.
# @LAYER: Core
# @RELATION: Aggregates models and manager.
# @RELATION: DEPENDS_ON ->[TaskManagerModels]
# @RELATION: DEPENDS_ON ->[TaskManagerModule]
# @RELATION: DEPENDS_ON ->[backend.src.core.task_manager.manager.TaskManager]
# @INVARIANT: Package exports stay aligned with manager and models contracts.
from .models import Task, TaskStatus, LogEntry
from .manager import TaskManager

2
backend/src/core/task_manager/__tests__/test_context.py
View File

@@ -1,5 +1,5 @@
# [DEF:backend.src.core.task_manager.__tests__.test_context:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: tests, task-context, background-tasks, sub-context
# @PURPOSE: Verify TaskContext preserves optional background task scheduler across sub-context creation.

4
backend/src/core/task_manager/cleanup.py
View File

@@ -1,5 +1,5 @@
# [DEF:TaskCleanupModule:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: task, cleanup, retention, logs
# @PURPOSE: Implements task cleanup and retention policies, including associated logs.
# @LAYER: Core
@@ -12,7 +12,7 @@ from ..config_manager import ConfigManager
# [DEF:TaskCleanupService:Class]
# @PURPOSE: Provides methods to clean up old task records and their associated logs.
# @TIER: STANDARD
# @COMPLEXITY: 3
class TaskCleanupService:
# [DEF:__init__:Function]
# @PURPOSE: Initializes the cleanup service with dependencies.

4
backend/src/core/task_manager/context.py
View File

@@ -3,7 +3,7 @@
# @PURPOSE: Provides execution context passed to plugins during task execution.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> TaskLogger, USED_BY -> plugins
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @INVARIANT: Each TaskContext is bound to a single task execution.
# [SECTION: IMPORTS]
@@ -16,7 +16,7 @@ from ..logger import belief_scope
# [DEF:TaskContext:Class]
# @SEMANTICS: context, task, execution, plugin
# @PURPOSE: A container passed to plugin.execute() providing the logger and other task-specific utilities.
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @INVARIANT: logger is always a valid TaskLogger instance.
# @UX_STATE: Idle -> Active -> Complete
#

48
backend/src/core/task_manager/manager.py
View File

@@ -1,5 +1,5 @@
# [DEF:TaskManagerModule:Module]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
# @LAYER: Core
@@ -38,7 +38,7 @@ from ..logger import logger, belief_scope, should_log_task_level
# [/SECTION]
# [DEF:TaskManager:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
# @INVARIANT: Task IDs are unique within the registry.
@@ -67,7 +67,7 @@ class TaskManager:
LOG_FLUSH_INTERVAL = 2.0
# [DEF:__init__:Function]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @PURPOSE: Initialize the TaskManager with dependencies.
# @PRE: plugin_loader is initialized.
# @POST: TaskManager is ready to accept tasks.
@@ -101,7 +101,7 @@ class TaskManager:
# [/DEF:__init__:Function]
# [DEF:_flusher_loop:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Background thread that periodically flushes log buffer to database.
# @PRE: TaskManager is initialized.
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
@@ -113,7 +113,7 @@ class TaskManager:
# [/DEF:_flusher_loop:Function]
# [DEF:_flush_logs:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Flush all buffered logs to the database.
# @PRE: None.
# @POST: All buffered logs are written to task_logs table.
@@ -140,7 +140,7 @@ class TaskManager:
# [/DEF:_flush_logs:Function]
# [DEF:_flush_task_logs:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Flush logs for a specific task immediately.
# @PRE: task_id exists.
# @POST: Task's buffered logs are written to database.
@@ -159,7 +159,7 @@ class TaskManager:
# [/DEF:_flush_task_logs:Function]
# [DEF:create_task:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Creates and queues a new task for execution.
# @PRE: Plugin with plugin_id exists. Params are valid.
# @POST: Task is created, added to registry, and scheduled for execution.
@@ -189,7 +189,7 @@ class TaskManager:
# [/DEF:create_task:Function]
# [DEF:_run_task:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Internal method to execute a task with TaskContext support.
# @PRE: Task exists in registry.
# @POST: Task is executed, status updated to SUCCESS or FAILED.
@@ -257,7 +257,7 @@ class TaskManager:
# [/DEF:_run_task:Function]
# [DEF:resolve_task:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Resumes a task that is awaiting mapping.
# @PRE: Task exists and is in AWAITING_MAPPING state.
# @POST: Task status updated to RUNNING, params updated, execution resumed.
@@ -282,7 +282,7 @@ class TaskManager:
# [/DEF:resolve_task:Function]
# [DEF:wait_for_resolution:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Pauses execution and waits for a resolution signal.
# @PRE: Task exists.
# @POST: Execution pauses until future is set.
@@ -305,7 +305,7 @@ class TaskManager:
# [/DEF:wait_for_resolution:Function]
# [DEF:wait_for_input:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Pauses execution and waits for user input.
# @PRE: Task exists.
# @POST: Execution pauses until future is set via resume_task_with_password.
@@ -327,7 +327,7 @@ class TaskManager:
# [/DEF:wait_for_input:Function]
# [DEF:get_task:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Retrieves a task by its ID.
# @PRE: task_id is a string.
# @POST: Returns Task object or None.
@@ -339,7 +339,7 @@ class TaskManager:
# [/DEF:get_task:Function]
# [DEF:get_all_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Retrieves all registered tasks.
# @PRE: None.
# @POST: Returns list of all Task objects.
@@ -350,7 +350,7 @@ class TaskManager:
# [/DEF:get_all_tasks:Function]
# [DEF:get_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Retrieves tasks with pagination and optional status filter.
# @PRE: limit and offset are non-negative integers.
# @POST: Returns a list of tasks sorted by start_time descending.
@@ -391,7 +391,7 @@ class TaskManager:
# [/DEF:get_tasks:Function]
# [DEF:get_task_logs:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).
# @PRE: task_id is a string.
# @POST: Returns list of LogEntry or TaskLog objects.
@@ -424,7 +424,7 @@ class TaskManager:
# [/DEF:get_task_logs:Function]
# [DEF:get_task_log_stats:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Get statistics about logs for a task.
# @PRE: task_id is a valid task ID.
# @POST: Returns LogStats with counts by level and source.
@@ -436,7 +436,7 @@ class TaskManager:
# [/DEF:get_task_log_stats:Function]
# [DEF:get_task_log_sources:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Get unique sources for a task's logs.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of unique source strings.
@@ -448,7 +448,7 @@ class TaskManager:
# [/DEF:get_task_log_sources:Function]
# [DEF:_add_log:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.
# @PRE: Task exists.
# @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).
@@ -501,7 +501,7 @@ class TaskManager:
# [/DEF:_add_log:Function]
# [DEF:subscribe_logs:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Subscribes to real-time logs for a task.
# @PRE: task_id is a string.
# @POST: Returns an asyncio.Queue for log entries.
@@ -517,7 +517,7 @@ class TaskManager:
# [/DEF:subscribe_logs:Function]
# [DEF:unsubscribe_logs:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Unsubscribes from real-time logs for a task.
# @PRE: task_id is a string, queue is asyncio.Queue.
# @POST: Queue removed from subscribers.
@@ -533,7 +533,7 @@ class TaskManager:
# [/DEF:unsubscribe_logs:Function]
# [DEF:load_persisted_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Load persisted tasks using persistence service.
# @PRE: None.
# @POST: Persisted tasks loaded into self.tasks.
@@ -546,7 +546,7 @@ class TaskManager:
# [/DEF:load_persisted_tasks:Function]
# [DEF:await_input:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Transition a task to AWAITING_INPUT state with input request.
# @PRE: Task exists and is in RUNNING state.
# @POST: Task status changed to AWAITING_INPUT, input_request set, persisted.
@@ -569,7 +569,7 @@ class TaskManager:
# [/DEF:await_input:Function]
# [DEF:resume_task_with_password:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Resume a task that is awaiting input with provided passwords.
# @PRE: Task exists and is in AWAITING_INPUT state.
# @POST: Task status changed to RUNNING, passwords injected, task resumed.
@@ -599,7 +599,7 @@ class TaskManager:
# [/DEF:resume_task_with_password:Function]
# [DEF:clear_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Clears tasks based on status filter (also deletes associated logs).
# @PRE: status is Optional[TaskStatus].
# @POST: Tasks matching filter (or all non-active) cleared from registry and database.

16
backend/src/core/task_manager/models.py
View File

@@ -1,5 +1,5 @@
# [DEF:TaskManagerModels:Module]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: task, models, pydantic, enum, state
# @PURPOSE: Defines the data models and enumerations used by the Task Manager.
# @LAYER: Core
@@ -17,7 +17,7 @@ from pydantic import BaseModel, Field
# [/SECTION]
# [DEF:TaskStatus:Enum]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @SEMANTICS: task, status, state, enum
# @PURPOSE: Defines the possible states a task can be in during its lifecycle.
class TaskStatus(str, Enum):
@@ -32,7 +32,7 @@ class TaskStatus(str, Enum):
# [DEF:LogLevel:Enum]
# @SEMANTICS: log, level, severity, enum
# @PURPOSE: Defines the possible log levels for task logging.
# @TIER: STANDARD
# @COMPLEXITY: 3
class LogLevel(str, Enum):
DEBUG = "DEBUG"
INFO = "INFO"
@@ -43,7 +43,7 @@ class LogLevel(str, Enum):
# [DEF:LogEntry:Class]
# @SEMANTICS: log, entry, record, pydantic
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @INVARIANT: Each log entry has a unique timestamp and source.
#
# @TEST_CONTRACT: LogEntryModel ->
@@ -65,7 +65,7 @@ class LogEntry(BaseModel):
# [DEF:TaskLog:Class]
# @SEMANTICS: task, log, persistent, pydantic
# @PURPOSE: A Pydantic model representing a persisted log entry from the database.
# @TIER: STANDARD
# @COMPLEXITY: 3
# @RELATION: MAPS_TO -> TaskLogRecord
class TaskLog(BaseModel):
id: int
@@ -83,7 +83,7 @@ class TaskLog(BaseModel):
# [DEF:LogFilter:Class]
# @SEMANTICS: log, filter, query, pydantic
# @PURPOSE: Filter parameters for querying task logs.
# @TIER: STANDARD
# @COMPLEXITY: 3
class LogFilter(BaseModel):
level: Optional[str] = None # Filter by log level
source: Optional[str] = None # Filter by source component
@@ -95,7 +95,7 @@ class LogFilter(BaseModel):
# [DEF:LogStats:Class]
# @SEMANTICS: log, stats, aggregation, pydantic
# @PURPOSE: Statistics about log entries for a task.
# @TIER: STANDARD
# @COMPLEXITY: 3
class LogStats(BaseModel):
total_count: int
by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2}
@@ -103,7 +103,7 @@ class LogStats(BaseModel):
# [/DEF:LogStats:Class]
# [DEF:Task:Class]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @SEMANTICS: task, job, execution, state, pydantic
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
class Task(BaseModel):

70
backend/src/core/task_manager/persistence.py
View File

@@ -1,5 +1,5 @@
# [DEF:TaskPersistenceModule:Module]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: persistence, sqlite, sqlalchemy, task, storage
# @PURPOSE: Handles the persistence of tasks using SQLAlchemy and the tasks.db database.
# @LAYER: Core
@@ -8,7 +8,7 @@
# @SIDE_EFFECT: Performs database I/O on tasks.db.
# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.database.TasksSessionLocal]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @INVARIANT: Database schema must match the TaskRecord model structure.
# [SECTION: IMPORTS]
@@ -26,12 +26,16 @@ from ..logger import logger, belief_scope
# [/SECTION]
# [DEF:TaskPersistenceService:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: persistence, service, database, sqlalchemy
# @PURPOSE: Provides methods to save and load tasks from the tasks.db database using SQLAlchemy.
# @RELATION: [DEPENDS_ON] ->[backend.src.core.database.TasksSessionLocal]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.task.TaskRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.mapping.Environment]
# @PURPOSE: Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models.
# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.
# @POST: Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.
# @SIDE_EFFECT: Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.
# @DATA_CONTRACT: Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [DEPENDS_ON] ->[TaskRecord]
# @RELATION: [DEPENDS_ON] ->[Environment]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @INVARIANT: Persistence must handle potentially missing task fields natively.
#
@@ -50,7 +54,7 @@ from ..logger import logger, belief_scope
# @TEST_INVARIANT: accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]
class TaskPersistenceService:
# [DEF:_json_load_if_needed:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Safely load JSON strings from DB if necessary
# @PRE: value is an arbitrary database value
# @POST: Returns parsed JSON object, list, string, or primitive
@@ -73,7 +77,7 @@ class TaskPersistenceService:
# [/DEF:_json_load_if_needed:Function]
# [DEF:_parse_datetime:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Safely parse a datetime string from the database
# @PRE: value is an ISO string or datetime object
# @POST: Returns datetime object or None
@@ -91,7 +95,7 @@ class TaskPersistenceService:
# [/DEF:_parse_datetime:Function]
# [DEF:_resolve_environment_id:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Resolve environment id into existing environments.id value to satisfy FK constraints.
# @PRE: Session is active
# @POST: Returns existing environments.id or None when unresolved.
@@ -130,7 +134,7 @@ class TaskPersistenceService:
# [/DEF:_resolve_environment_id:Function]
# [DEF:__init__:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Initializes the persistence service.
# @PRE: None.
# @POST: Service is ready.
@@ -141,14 +145,14 @@ class TaskPersistenceService:
# [/DEF:__init__:Function]
# [DEF:persist_task:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Persists or updates a single task in the database.
# @PRE: isinstance(task, Task)
# @POST: Task record created or updated in database.
# @PARAM: task (Task) - The task object to persist.
# @SIDE_EFFECT: Writes to task_records table in tasks.db
# @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]
# @RELATION: [CALLS] ->[self._resolve_environment_id]
# @RELATION: [CALLS] ->[_resolve_environment_id]
def persist_task(self, task: Task) -> None:
with belief_scope("TaskPersistenceService.persist_task", f"task_id={task.id}"):
session: Session = TasksSessionLocal()
@@ -206,12 +210,12 @@ class TaskPersistenceService:
# [/DEF:persist_task:Function]
# [DEF:persist_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Persists multiple tasks.
# @PRE: isinstance(tasks, list)
# @POST: All tasks in list are persisted.
# @PARAM: tasks (List[Task]) - The list of tasks to persist.
# @RELATION: [CALLS] ->[self.persist_task]
# @RELATION: [CALLS] ->[persist_task]
def persist_tasks(self, tasks: List[Task]) -> None:
with belief_scope("TaskPersistenceService.persist_tasks"):
for task in tasks:
@@ -219,7 +223,7 @@ class TaskPersistenceService:
# [/DEF:persist_tasks:Function]
# [DEF:load_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Loads tasks from the database.
# @PRE: limit is an integer.
# @POST: Returns list of Task objects.
@@ -227,8 +231,8 @@ class TaskPersistenceService:
# @PARAM: status (Optional[TaskStatus]) - Filter by status.
# @RETURN: List[Task] - The loaded tasks.
# @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]
# @RELATION: [CALLS] ->[self._json_load_if_needed]
# @RELATION: [CALLS] ->[self._parse_datetime]
# @RELATION: [CALLS] ->[_json_load_if_needed]
# @RELATION: [CALLS] ->[_parse_datetime]
def load_tasks(self, limit: int = 100, status: Optional[TaskStatus] = None) -> List[Task]:
with belief_scope("TaskPersistenceService.load_tasks"):
session: Session = TasksSessionLocal()
@@ -277,7 +281,7 @@ class TaskPersistenceService:
# [/DEF:load_tasks:Function]
# [DEF:delete_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Deletes specific tasks from the database.
# @PRE: task_ids is a list of strings.
# @POST: Specified task records deleted from database.
@@ -297,15 +301,18 @@ class TaskPersistenceService:
finally:
session.close()
# [/DEF:delete_tasks:Function]
# [/DEF:TaskPersistenceService:Class]
# [DEF:TaskLogPersistenceService:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: persistence, service, database, log, sqlalchemy
# @PURPOSE: Provides methods to save and query task logs from the task_logs table.
# @RELATION: [DEPENDS_ON] ->[backend.src.models.task.TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.database.TasksSessionLocal]
# @PURPOSE: Provides methods to store, query, summarize, and delete task log rows in the task_logs table.
# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.
# @POST: add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers.
# @SIDE_EFFECT: Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures.
# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @INVARIANT: Log entries are batch-inserted for performance.
#
@@ -328,7 +335,7 @@ class TaskLogPersistenceService:
"""
# [DEF:__init__:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Initializes the TaskLogPersistenceService
# @PRE: config is provided or defaults are used
# @POST: Service is ready for log persistence
@@ -337,7 +344,7 @@ class TaskLogPersistenceService:
# [/DEF:__init__:Function]
# [DEF:add_logs:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Batch insert log entries for a task.
# @PRE: logs is a list of LogEntry objects.
# @POST: All logs inserted into task_logs table.
@@ -370,7 +377,7 @@ class TaskLogPersistenceService:
# [/DEF:add_logs:Function]
# [DEF:get_logs:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Query logs for a task with filtering and pagination.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of TaskLog objects matching filters.
@@ -424,7 +431,7 @@ class TaskLogPersistenceService:
# [/DEF:get_logs:Function]
# [DEF:get_log_stats:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Get statistics about logs for a task.
# @PRE: task_id is a valid task ID.
# @POST: Returns LogStats with counts by level and source.
@@ -471,7 +478,7 @@ class TaskLogPersistenceService:
# [/DEF:get_log_stats:Function]
# [DEF:get_sources:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Get unique sources for a task's logs.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of unique source strings.
@@ -492,7 +499,7 @@ class TaskLogPersistenceService:
# [/DEF:get_sources:Function]
# [DEF:delete_logs_for_task:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Delete all logs for a specific task.
# @PRE: task_id is a valid task ID.
# @POST: All logs for the task are deleted.
@@ -514,7 +521,7 @@ class TaskLogPersistenceService:
# [/DEF:delete_logs_for_task:Function]
# [DEF:delete_logs_for_tasks:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Delete all logs for multiple tasks.
# @PRE: task_ids is a list of task IDs.
# @POST: All logs for the tasks are deleted.
@@ -536,6 +543,5 @@ class TaskLogPersistenceService:
finally:
session.close()
# [/DEF:delete_logs_for_tasks:Function]
# [/DEF:TaskLogPersistenceService:Class]
# [/DEF:TaskPersistenceModule:Module]

4
backend/src/core/task_manager/task_logger.py
View File

@@ -3,7 +3,7 @@
# @PURPOSE: Provides a dedicated logger for tasks with automatic source attribution.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> TaskManager, CALLS -> TaskManager._add_log
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source.
# [SECTION: IMPORTS]
@@ -13,7 +13,7 @@ from typing import Dict, Any, Optional, Callable
# [DEF:TaskLogger:Class]
# @SEMANTICS: logger, task, source, attribution
# @PURPOSE: A wrapper around TaskManager._add_log that carries task_id and source context.
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @INVARIANT: All log calls include the task_id and source.
# @UX_STATE: Idle -> Logging -> (system records log)
#

34
backend/src/core/utils/async_network.py
View File

@@ -1,9 +1,13 @@
# [DEF:backend.src.core.utils.async_network:Module]
#
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: network, httpx, async, superset, authentication, cache
# @PURPOSE: Provides async Superset API client with shared auth-token cache to avoid per-request re-login.
# @LAYER: Infra
# @PRE: Config payloads contain a Superset base URL and authentication fields needed for login.
# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @RELATION: DEPENDS_ON -> backend.src.core.utils.network.SupersetAuthCache
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
@@ -25,8 +29,8 @@ from .network import (
# [/SECTION]
# [DEF:AsyncAPIClient:Class]
# @TIER: STANDARD
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient:Class]
# @COMPLEXITY: 3
# @PURPOSE: Async Superset API client backed by httpx.AsyncClient with shared auth cache.
# @RELATION: [DEPENDS_ON] ->[backend.src.core.utils.network.SupersetAuthCache]
# @RELATION: [CALLS] ->[backend.src.core.utils.network.SupersetAuthCache.get]
@@ -35,8 +39,8 @@ class AsyncAPIClient:
DEFAULT_TIMEOUT = 30
_auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}
# [DEF:__init__:Function]
# @TIER: STANDARD
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient.__init__:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initialize async API client for one environment.
# @PRE: config contains base_url and auth payload.
# @POST: Client is ready for async request/authentication flow.
@@ -61,8 +65,8 @@ class AsyncAPIClient:
# [/DEF:__init__:Function]
# [DEF:_normalize_base_url:Function]
# @TIER: TRIVIAL
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient._normalize_base_url:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize base URL for Superset API root construction.
# @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
def _normalize_base_url(self, raw_url: str) -> str:
@@ -73,7 +77,7 @@ class AsyncAPIClient:
# [/DEF:_normalize_base_url:Function]
# [DEF:_build_api_url:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Build full API URL from relative Superset endpoint.
# @POST: Returns absolute URL for upstream request.
def _build_api_url(self, endpoint: str) -> str:
@@ -88,7 +92,7 @@ class AsyncAPIClient:
# [/DEF:_build_api_url:Function]
# [DEF:_get_auth_lock:Function]
# @TIER: TRIVIAL
# @COMPLEXITY: 1
# @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.
# @POST: Returns stable asyncio.Lock instance.
@classmethod
@@ -102,7 +106,7 @@ class AsyncAPIClient:
# [/DEF:_get_auth_lock:Function]
# [DEF:authenticate:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Authenticate against Superset and cache access/csrf tokens.
# @POST: Client tokens are populated and reusable across requests.
# @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.
@@ -162,7 +166,7 @@ class AsyncAPIClient:
# [/DEF:authenticate:Function]
# [DEF:get_headers:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Return authenticated Superset headers for async requests.
# @POST: Headers include Authorization and CSRF tokens.
# @RELATION: CALLS -> self.authenticate
@@ -178,7 +182,7 @@ class AsyncAPIClient:
# [/DEF:get_headers:Function]
# [DEF:request:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Perform one authenticated async Superset API request.
# @POST: Returns JSON payload or raw httpx.Response when raw_response=true.
# @SIDE_EFFECT: Performs network I/O.
@@ -215,7 +219,7 @@ class AsyncAPIClient:
# [/DEF:request:Function]
# [DEF:_handle_http_error:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.
# @POST: Raises domain-specific exception for caller flow control.
# @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception
@@ -234,7 +238,7 @@ class AsyncAPIClient:
# [/DEF:_handle_http_error:Function]
# [DEF:_handle_network_error:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Translate generic httpx errors into NetworkError.
# @POST: Raises NetworkError with URL context.
# @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError
@@ -250,7 +254,7 @@ class AsyncAPIClient:
# [/DEF:_handle_network_error:Function]
# [DEF:aclose:Function]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Close underlying httpx client.
# @POST: Client resources are released.
# @SIDE_EFFECT: Closes network connections.

17
backend/src/core/utils/network.py
View File

@@ -1,5 +1,6 @@
# [DEF:backend.core.utils.network:Module]
# [DEF:network:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: network, http, client, api, requests, session, authentication
# @PURPOSE: Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
# @LAYER: Infra
@@ -22,9 +23,11 @@ from ..logger import logger as app_logger, belief_scope
# [/SECTION]
# [DEF:SupersetAPIError:Class]
# @COMPLEXITY: 1
# @PURPOSE: Base exception for all Superset API related errors.
class SupersetAPIError(Exception):
# [DEF:__init__:Function]
# @COMPLEXITY: 1
# @PURPOSE: Initializes the exception with a message and context.
# @PRE: message is a string, context is a dict.
# @POST: Exception is initialized with context.
@@ -36,9 +39,11 @@ class SupersetAPIError(Exception):
# [/DEF:SupersetAPIError:Class]
# [DEF:AuthenticationError:Class]
# @COMPLEXITY: 1
# @PURPOSE: Exception raised when authentication fails.
class AuthenticationError(SupersetAPIError):
# [DEF:__init__:Function]
# @COMPLEXITY: 1
# @PURPOSE: Initializes the authentication error.
# @PRE: message is a string, context is a dict.
# @POST: AuthenticationError is initialized.
@@ -77,7 +82,7 @@ class DashboardNotFoundError(SupersetAPIError):
# [DEF:NetworkError:Class]
# @PURPOSE: Exception raised when a network level error occurs.
class NetworkError(Exception):
# [DEF:__init__:Function]
# [DEF:network.APIClient.__init__:Function]
# @PURPOSE: Initializes the network error.
# @PRE: message is a string.
# @POST: NetworkError is initialized.
@@ -89,7 +94,7 @@ class NetworkError(Exception):
# [/DEF:NetworkError:Class]
# [DEF:SupersetAuthCache:Class]
# [DEF:network.SupersetAuthCache:Class]
# @PURPOSE: Process-local cache for Superset access/csrf tokens keyed by environment credentials.
# @PRE: base_url and username are stable strings.
# @POST: Cached entries expire automatically by TTL and can be reused across requests.
@@ -145,10 +150,10 @@ class SupersetAuthCache:
# [/DEF:SupersetAuthCache:Class]
# [DEF:APIClient:Class]
# @TIER: STANDARD
# @COMPLEXITY: 3
# @PURPOSE: Synchronous Superset API client with process-local auth token caching.
# @RELATION: DEPENDS_ON -> backend.src.core.utils.network.SupersetAuthCache
# @RELATION: DEPENDS_ON -> backend.src.core.logger.logger
# @RELATION: DEPENDS_ON -> network.SupersetAuthCache
# @RELATION: DEPENDS_ON -> logger
class APIClient:
DEFAULT_TIMEOUT = 30