diff --git a/backend/src/api/auth.py b/backend/src/api/auth.py index 014cb992..4aa8be14 100755 --- a/backend/src/api/auth.py +++ b/backend/src/api/auth.py @@ -1,118 +1,125 @@ -# [DEF:backend.src.api.auth:Module] -# -# @SEMANTICS: api, auth, routes, login, logout -# @PURPOSE: Authentication API endpoints. -# @LAYER: API -# @RELATION: USES -> backend.src.services.auth_service.AuthService -# @RELATION: USES -> backend.src.core.database.get_auth_db -# -# @INVARIANT: All auth endpoints must return consistent error codes. - -# [SECTION: IMPORTS] -from fastapi import APIRouter, Depends, HTTPException, status -from fastapi.security import OAuth2PasswordRequestForm -from sqlalchemy.orm import Session -from ..core.database import get_auth_db -from ..services.auth_service import AuthService -from ..schemas.auth import Token, User as UserSchema -from ..dependencies import get_current_user -from ..core.auth.oauth import oauth, is_adfs_configured -from ..core.auth.logger import log_security_event -from ..core.logger import belief_scope -import starlette.requests -# [/SECTION] - -# [DEF:router:Variable] -# @PURPOSE: APIRouter instance for authentication routes. -router = APIRouter(prefix="/api/auth", tags=["auth"]) -# [/DEF:router:Variable] - -# [DEF:login_for_access_token:Function] -# @PURPOSE: Authenticates a user and returns a JWT access token. -# @PRE: form_data contains username and password. -# @POST: Returns a Token object on success. -# @THROW: HTTPException 401 if authentication fails. -# @PARAM: form_data (OAuth2PasswordRequestForm) - Login credentials. -# @PARAM: db (Session) - Auth database session. -# @RETURN: Token - The generated JWT token. -@router.post("/login", response_model=Token) -async def login_for_access_token( - form_data: OAuth2PasswordRequestForm = Depends(), - db: Session = Depends(get_auth_db) -): - with belief_scope("api.auth.login"): - auth_service = AuthService(db) - user = auth_service.authenticate_user(form_data.username, form_data.password) - if not user: - log_security_event("LOGIN_FAILED", form_data.username, {"reason": "Invalid credentials"}) - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Incorrect username or password", - headers={"WWW-Authenticate": "Bearer"}, - ) - log_security_event("LOGIN_SUCCESS", user.username, {"source": "LOCAL"}) - return auth_service.create_session(user) -# [/DEF:login_for_access_token:Function] - -# [DEF:read_users_me:Function] -# @PURPOSE: Retrieves the profile of the currently authenticated user. -# @PRE: Valid JWT token provided. -# @POST: Returns the current user's data. -# @PARAM: current_user (UserSchema) - The user extracted from the token. -# @RETURN: UserSchema - The current user profile. -@router.get("/me", response_model=UserSchema) -async def read_users_me(current_user: UserSchema = Depends(get_current_user)): - with belief_scope("api.auth.me"): - return current_user -# [/DEF:read_users_me:Function] - -# [DEF:logout:Function] -# @PURPOSE: Logs out the current user (placeholder for session revocation). -# @PRE: Valid JWT token provided. -# @POST: Returns success message. -@router.post("/logout") -async def logout(current_user: UserSchema = Depends(get_current_user)): - with belief_scope("api.auth.logout"): - log_security_event("LOGOUT", current_user.username) - # In a stateless JWT setup, client-side token deletion is primary. - # Server-side revocation (blacklisting) can be added here if needed. - return {"message": "Successfully logged out"} -# [/DEF:logout:Function] - -# [DEF:login_adfs:Function] -# @PURPOSE: Initiates the ADFS OIDC login flow. -# @POST: Redirects the user to ADFS. -@router.get("/login/adfs") -async def login_adfs(request: starlette.requests.Request): - with belief_scope("api.auth.login_adfs"): - if not is_adfs_configured(): - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="ADFS is not configured. Please set ADFS_CLIENT_ID, ADFS_CLIENT_SECRET, and ADFS_METADATA_URL environment variables." - ) - redirect_uri = request.url_for('auth_callback_adfs') - return await oauth.adfs.authorize_redirect(request, str(redirect_uri)) -# [/DEF:login_adfs:Function] - -# [DEF:auth_callback_adfs:Function] -# @PURPOSE: Handles the callback from ADFS after successful authentication. -# @POST: Provisions user JIT and returns session token. -@router.get("/callback/adfs", name="auth_callback_adfs") -async def auth_callback_adfs(request: starlette.requests.Request, db: Session = Depends(get_auth_db)): - with belief_scope("api.auth.callback_adfs"): - if not is_adfs_configured(): - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="ADFS is not configured. Please set ADFS_CLIENT_ID, ADFS_CLIENT_SECRET, and ADFS_METADATA_URL environment variables." - ) - token = await oauth.adfs.authorize_access_token(request) - user_info = token.get('userinfo') - if not user_info: - raise HTTPException(status_code=400, detail="Failed to retrieve user info from ADFS") - - auth_service = AuthService(db) - user = auth_service.provision_adfs_user(user_info) - return auth_service.create_session(user) -# [/DEF:auth_callback_adfs:Function] - +# [DEF:backend.src.api.auth:Module] +# +# @TIER: STANDARD +# @SEMANTICS: api, auth, routes, login, logout +# @PURPOSE: Authentication API endpoints. +# @LAYER: API +# @RELATION: USES ->[backend.src.services.auth_service.AuthService] +# @RELATION: USES ->[backend.src.core.database.get_auth_db] +# +# @INVARIANT: All auth endpoints must return consistent error codes. + +# [SECTION: IMPORTS] +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import OAuth2PasswordRequestForm +from sqlalchemy.orm import Session +from ..core.database import get_auth_db +from ..services.auth_service import AuthService +from ..schemas.auth import Token, User as UserSchema +from ..dependencies import get_current_user +from ..core.auth.oauth import oauth, is_adfs_configured +from ..core.auth.logger import log_security_event +from ..core.logger import belief_scope +import starlette.requests +# [/SECTION] + +# [DEF:router:Variable] +# @TIER: TRIVIAL +# @PURPOSE: APIRouter instance for authentication routes. +router = APIRouter(prefix="/api/auth", tags=["auth"]) +# [/DEF:router:Variable] + +# [DEF:login_for_access_token:Function] +# @TIER: STANDARD +# @PURPOSE: Authenticates a user and returns a JWT access token. +# @PRE: form_data contains username and password. +# @POST: Returns a Token object on success. +# @THROW: HTTPException 401 if authentication fails. +# @PARAM: form_data (OAuth2PasswordRequestForm) - Login credentials. +# @PARAM: db (Session) - Auth database session. +# @RETURN: Token - The generated JWT token. +@router.post("/login", response_model=Token) +async def login_for_access_token( + form_data: OAuth2PasswordRequestForm = Depends(), + db: Session = Depends(get_auth_db) +): + with belief_scope("api.auth.login"): + auth_service = AuthService(db) + user = auth_service.authenticate_user(form_data.username, form_data.password) + if not user: + log_security_event("LOGIN_FAILED", form_data.username, {"reason": "Invalid credentials"}) + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect username or password", + headers={"WWW-Authenticate": "Bearer"}, + ) + log_security_event("LOGIN_SUCCESS", user.username, {"source": "LOCAL"}) + return auth_service.create_session(user) +# [/DEF:login_for_access_token:Function] + +# [DEF:read_users_me:Function] +# @TIER: STANDARD +# @PURPOSE: Retrieves the profile of the currently authenticated user. +# @PRE: Valid JWT token provided. +# @POST: Returns the current user's data. +# @PARAM: current_user (UserSchema) - The user extracted from the token. +# @RETURN: UserSchema - The current user profile. +@router.get("/me", response_model=UserSchema) +async def read_users_me(current_user: UserSchema = Depends(get_current_user)): + with belief_scope("api.auth.me"): + return current_user +# [/DEF:read_users_me:Function] + +# [DEF:logout:Function] +# @TIER: STANDARD +# @PURPOSE: Logs out the current user (placeholder for session revocation). +# @PRE: Valid JWT token provided. +# @POST: Returns success message. +@router.post("/logout") +async def logout(current_user: UserSchema = Depends(get_current_user)): + with belief_scope("api.auth.logout"): + log_security_event("LOGOUT", current_user.username) + # In a stateless JWT setup, client-side token deletion is primary. + # Server-side revocation (blacklisting) can be added here if needed. + return {"message": "Successfully logged out"} +# [/DEF:logout:Function] + +# [DEF:login_adfs:Function] +# @TIER: STANDARD +# @PURPOSE: Initiates the ADFS OIDC login flow. +# @POST: Redirects the user to ADFS. +@router.get("/login/adfs") +async def login_adfs(request: starlette.requests.Request): + with belief_scope("api.auth.login_adfs"): + if not is_adfs_configured(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="ADFS is not configured. Please set ADFS_CLIENT_ID, ADFS_CLIENT_SECRET, and ADFS_METADATA_URL environment variables." + ) + redirect_uri = request.url_for('auth_callback_adfs') + return await oauth.adfs.authorize_redirect(request, str(redirect_uri)) +# [/DEF:login_adfs:Function] + +# [DEF:auth_callback_adfs:Function] +# @TIER: STANDARD +# @PURPOSE: Handles the callback from ADFS after successful authentication. +# @POST: Provisions user JIT and returns session token. +@router.get("/callback/adfs", name="auth_callback_adfs") +async def auth_callback_adfs(request: starlette.requests.Request, db: Session = Depends(get_auth_db)): + with belief_scope("api.auth.callback_adfs"): + if not is_adfs_configured(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="ADFS is not configured. Please set ADFS_CLIENT_ID, ADFS_CLIENT_SECRET, and ADFS_METADATA_URL environment variables." + ) + token = await oauth.adfs.authorize_access_token(request) + user_info = token.get('userinfo') + if not user_info: + raise HTTPException(status_code=400, detail="Failed to retrieve user info from ADFS") + + auth_service = AuthService(db) + user = auth_service.provision_adfs_user(user_info) + return auth_service.create_session(user) +# [/DEF:auth_callback_adfs:Function] + # [/DEF:backend.src.api.auth:Module] \ No newline at end of file diff --git a/backend/src/api/routes/admin.py b/backend/src/api/routes/admin.py index 5af2125c..bb0359cc 100644 --- a/backend/src/api/routes/admin.py +++ b/backend/src/api/routes/admin.py @@ -4,8 +4,8 @@ # @SEMANTICS: api, admin, users, roles, permissions # @PURPOSE: Admin API endpoints for user and role management. # @LAYER: API -# @RELATION: USES -> backend.src.core.auth.repository.AuthRepository -# @RELATION: USES -> backend.src.dependencies.has_permission +# @RELATION: [USES] ->[backend.src.core.auth.repository.AuthRepository] +# @RELATION: [USES] ->[backend.src.dependencies.has_permission] # # @INVARIANT: All endpoints in this module require 'Admin' role or 'admin' scope. @@ -36,6 +36,7 @@ router = APIRouter(prefix="/api/admin", tags=["admin"]) # [/DEF:router:Variable] # [DEF:list_users:Function] +# @TIER: STANDARD # @PURPOSE: Lists all registered users. # @PRE: Current user has 'Admin' role. # @POST: Returns a list of UserSchema objects. @@ -52,6 +53,7 @@ async def list_users( # [/DEF:list_users:Function] # [DEF:create_user:Function] +# @TIER: STANDARD # @PURPOSE: Creates a new local user. # @PRE: Current user has 'Admin' role. # @POST: New user is created in the database. @@ -89,6 +91,7 @@ async def create_user( # [/DEF:create_user:Function] # [DEF:update_user:Function] +# @TIER: STANDARD # @PURPOSE: Updates an existing user. @router.put("/users/{user_id}", response_model=UserSchema) async def update_user( @@ -123,6 +126,7 @@ async def update_user( # [/DEF:update_user:Function] # [DEF:delete_user:Function] +# @TIER: STANDARD # @PURPOSE: Deletes a user. @router.delete("/users/{user_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_user( @@ -146,6 +150,7 @@ async def delete_user( # [/DEF:delete_user:Function] # [DEF:list_roles:Function] +# @TIER: STANDARD # @PURPOSE: Lists all available roles. # @RETURN: List[RoleSchema] - List of roles. # @RELATION: CALLS -> backend.src.models.auth.Role @@ -159,6 +164,7 @@ async def list_roles( # [/DEF:list_roles:Function] # [DEF:create_role:Function] +# @TIER: STANDARD # @PURPOSE: Creates a new system role with associated permissions. # @PRE: Role name must be unique. # @POST: New Role record is created in auth.db. @@ -196,6 +202,7 @@ async def create_role( # [/DEF:create_role:Function] # [DEF:update_role:Function] +# @TIER: STANDARD # @PURPOSE: Updates an existing role's metadata and permissions. # @PRE: role_id must be a valid existing role UUID. # @POST: Role record is updated in auth.db. @@ -240,6 +247,7 @@ async def update_role( # [/DEF:update_role:Function] # [DEF:delete_role:Function] +# @TIER: STANDARD # @PURPOSE: Removes a role from the system. # @PRE: role_id must be a valid existing role UUID. # @POST: Role record is removed from auth.db. @@ -266,6 +274,7 @@ async def delete_role( # [/DEF:delete_role:Function] # [DEF:list_permissions:Function] +# @TIER: STANDARD # @PURPOSE: Lists all available system permissions for assignment. # @POST: Returns a list of all PermissionSchema objects. # @PARAM: db (Session) - Auth database session. @@ -291,6 +300,7 @@ async def list_permissions( # [/DEF:list_permissions:Function] # [DEF:list_ad_mappings:Function] +# @TIER: STANDARD # @PURPOSE: Lists all AD Group to Role mappings. @router.get("/ad-mappings", response_model=List[ADGroupMappingSchema]) async def list_ad_mappings( @@ -302,6 +312,7 @@ async def list_ad_mappings( # [/DEF:list_ad_mappings:Function] # [DEF:create_ad_mapping:Function] +# @TIER: STANDARD # @PURPOSE: Creates a new AD Group mapping. @router.post("/ad-mappings", response_model=ADGroupMappingSchema) async def create_ad_mapping( diff --git a/backend/src/api/routes/assistant.py b/backend/src/api/routes/assistant.py index 28d54510..4f525f63 100644 --- a/backend/src/api/routes/assistant.py +++ b/backend/src/api/routes/assistant.py @@ -3,8 +3,8 @@ # @SEMANTICS: api, assistant, chat, command, confirmation # @PURPOSE: API routes for LLM assistant command parsing and safe execution orchestration. # @LAYER: API -# @RELATION: DEPENDS_ON -> backend.src.core.task_manager -# @RELATION: DEPENDS_ON -> backend.src.models.assistant +# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.manager.TaskManager] +# @RELATION: [DEPENDS_ON] ->[backend.src.models.assistant] # @INVARIANT: Risky operations are never executed without valid confirmation token. from __future__ import annotations @@ -125,6 +125,7 @@ INTENT_PERMISSION_CHECKS: Dict[str, List[Tuple[str, str]]] = { # [DEF:_append_history:Function] +# @TIER: STANDARD # @PURPOSE: Append conversation message to in-memory history buffer. # @PRE: user_id and conversation_id identify target conversation bucket. # @POST: Message entry is appended to CONVERSATIONS key list. @@ -156,6 +157,7 @@ def _append_history( # [DEF:_persist_message:Function] +# @TIER: STANDARD # @PURPOSE: Persist assistant/user message record to database. # @PRE: db session is writable and message payload is serializable. # @POST: Message row is committed or persistence failure is logged. @@ -191,6 +193,7 @@ def _persist_message( # [DEF:_audit:Function] +# @TIER: STANDARD # @PURPOSE: Append in-memory audit record for assistant decision trace. # @PRE: payload describes decision/outcome fields. # @POST: ASSISTANT_AUDIT list for user contains new timestamped entry. @@ -203,6 +206,7 @@ def _audit(user_id: str, payload: Dict[str, Any]): # [DEF:_persist_audit:Function] +# @TIER: STANDARD # @PURPOSE: Persist structured assistant audit payload in database. # @PRE: db session is writable and payload is JSON-serializable. # @POST: Audit row is committed or failure is logged with rollback. @@ -226,6 +230,7 @@ def _persist_audit(db: Session, user_id: str, payload: Dict[str, Any], conversat # [DEF:_persist_confirmation:Function] +# @TIER: STANDARD # @PURPOSE: Persist confirmation token record to database. # @PRE: record contains id/user/intent/dispatch/expiry fields. # @POST: Confirmation row exists in persistent storage. @@ -251,6 +256,7 @@ def _persist_confirmation(db: Session, record: ConfirmationRecord): # [DEF:_update_confirmation_state:Function] +# @TIER: STANDARD # @PURPOSE: Update persistent confirmation token lifecycle state. # @PRE: confirmation_id references existing row. # @POST: State and consumed_at fields are updated when applicable. @@ -270,6 +276,7 @@ def _update_confirmation_state(db: Session, confirmation_id: str, state: str): # [DEF:_load_confirmation_from_db:Function] +# @TIER: STANDARD # @PURPOSE: Load confirmation token from database into in-memory model. # @PRE: confirmation_id may or may not exist in storage. # @POST: Returns ConfirmationRecord when found, otherwise None. @@ -295,6 +302,7 @@ def _load_confirmation_from_db(db: Session, confirmation_id: str) -> Optional[Co # [DEF:_ensure_conversation:Function] +# @TIER: STANDARD # @PURPOSE: Resolve active conversation id in memory or create a new one. # @PRE: user_id identifies current actor. # @POST: Returns stable conversation id and updates USER_ACTIVE_CONVERSATION. @@ -314,6 +322,7 @@ def _ensure_conversation(user_id: str, conversation_id: Optional[str]) -> str: # [DEF:_resolve_or_create_conversation:Function] +# @TIER: STANDARD # @PURPOSE: Resolve active conversation using explicit id, memory cache, or persisted history. # @PRE: user_id and db session are available. # @POST: Returns conversation id and updates USER_ACTIVE_CONVERSATION cache. @@ -343,6 +352,7 @@ def _resolve_or_create_conversation(user_id: str, conversation_id: Optional[str] # [DEF:_cleanup_history_ttl:Function] +# @TIER: STANDARD # @PURPOSE: Enforce assistant message retention window by deleting expired rows and in-memory records. # @PRE: db session is available and user_id references current actor scope. # @POST: Messages older than ASSISTANT_MESSAGE_TTL_DAYS are removed from persistence and memory mirrors. @@ -380,6 +390,7 @@ def _cleanup_history_ttl(db: Session, user_id: str): # [DEF:_is_conversation_archived:Function] +# @TIER: STANDARD # @PURPOSE: Determine archived state for a conversation based on last update timestamp. # @PRE: updated_at can be null for empty conversations. # @POST: Returns True when conversation inactivity exceeds archive threshold. @@ -392,6 +403,7 @@ def _is_conversation_archived(updated_at: Optional[datetime]) -> bool: # [DEF:_coerce_query_bool:Function] +# @TIER: STANDARD # @PURPOSE: Normalize bool-like query values for compatibility in direct handler invocations/tests. # @PRE: value may be bool, string, or FastAPI Query metadata object. # @POST: Returns deterministic boolean flag. @@ -405,6 +417,7 @@ def _coerce_query_bool(value: Any) -> bool: # [DEF:_extract_id:Function] +# @TIER: STANDARD # @PURPOSE: Extract first regex match group from text by ordered pattern list. # @PRE: patterns contain at least one capture group. # @POST: Returns first matched token or None. @@ -418,6 +431,7 @@ def _extract_id(text: str, patterns: List[str]) -> Optional[str]: # [DEF:_resolve_env_id:Function] +# @TIER: STANDARD # @PURPOSE: Resolve environment identifier/name token to canonical environment id. # @PRE: config_manager provides environment list. # @POST: Returns matched environment id or None. @@ -435,6 +449,7 @@ def _resolve_env_id(token: Optional[str], config_manager: ConfigManager) -> Opti # [DEF:_is_production_env:Function] +# @TIER: STANDARD # @PURPOSE: Determine whether environment token resolves to production-like target. # @PRE: config_manager provides environments or token text is provided. # @POST: Returns True for production/prod synonyms, else False. @@ -452,6 +467,7 @@ def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> b # [DEF:_resolve_provider_id:Function] +# @TIER: STANDARD # @PURPOSE: Resolve provider token to provider id with active/default fallback. # @PRE: db session can load provider list through LLMProviderService. # @POST: Returns provider id or None when no providers configured. @@ -487,6 +503,7 @@ def _resolve_provider_id( # [DEF:_get_default_environment_id:Function] +# @TIER: STANDARD # @PURPOSE: Resolve default environment id from settings or first configured environment. # @PRE: config_manager returns environments list. # @POST: Returns default environment id or None when environment list is empty. @@ -508,6 +525,7 @@ def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: # [DEF:_resolve_dashboard_id_by_ref:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard id by title or slug reference in selected environment. # @PRE: dashboard_ref is a non-empty string-like token. # @POST: Returns dashboard id when uniquely matched, otherwise None. @@ -550,6 +568,7 @@ def _resolve_dashboard_id_by_ref( # [DEF:_resolve_dashboard_id_entity:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard id from intent entities using numeric id or dashboard_ref fallback. # @PRE: entities may contain dashboard_id as int/str and optional dashboard_ref. # @POST: Returns resolved dashboard id or None when ambiguous/unresolvable. @@ -581,6 +600,7 @@ def _resolve_dashboard_id_entity( # [DEF:_get_environment_name_by_id:Function] +# @TIER: STANDARD # @PURPOSE: Resolve human-readable environment name by id. # @PRE: environment id may be None. # @POST: Returns matching environment name or fallback id. @@ -593,6 +613,7 @@ def _get_environment_name_by_id(env_id: Optional[str], config_manager: ConfigMan # [DEF:_extract_result_deep_links:Function] +# @TIER: STANDARD # @PURPOSE: Build deep-link actions to verify task result from assistant chat. # @PRE: task object is available. # @POST: Returns zero or more assistant actions for dashboard open/diff. @@ -649,6 +670,7 @@ def _extract_result_deep_links(task: Any, config_manager: ConfigManager) -> List # [DEF:_build_task_observability_summary:Function] +# @TIER: STANDARD # @PURPOSE: Build compact textual summary for completed tasks to reduce "black box" effect. # @PRE: task may contain plugin-specific result payload. # @POST: Returns non-empty summary line for known task types or empty string fallback. @@ -712,6 +734,7 @@ def _build_task_observability_summary(task: Any, config_manager: ConfigManager) # [DEF:_parse_command:Function] +# @TIER: STANDARD # @PURPOSE: Deterministically parse RU/EN command text into intent payload. # @PRE: message contains raw user text and config manager resolves environments. # @POST: Returns intent dict with domain/operation/entities/confidence/risk fields. @@ -905,6 +928,7 @@ def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any # [DEF:_check_any_permission:Function] +# @TIER: STANDARD # @PURPOSE: Validate user against alternative permission checks (logical OR). # @PRE: checks list contains resource-action tuples. # @POST: Returns on first successful permission; raises 403-like HTTPException otherwise. @@ -922,6 +946,7 @@ def _check_any_permission(current_user: User, checks: List[Tuple[str, str]]): # [DEF:_has_any_permission:Function] +# @TIER: STANDARD # @PURPOSE: Check whether user has at least one permission tuple from the provided list. # @PRE: current_user and checks list are valid. # @POST: Returns True when at least one permission check passes. @@ -935,6 +960,7 @@ def _has_any_permission(current_user: User, checks: List[Tuple[str, str]]) -> bo # [DEF:_build_tool_catalog:Function] +# @TIER: STANDARD # @PURPOSE: Build current-user tool catalog for LLM planner with operation contracts and defaults. # @PRE: current_user is authenticated; config/db are available. # @POST: Returns list of executable tools filtered by permission and runtime availability. @@ -1058,6 +1084,7 @@ def _build_tool_catalog(current_user: User, config_manager: ConfigManager, db: S # [DEF:_coerce_intent_entities:Function] +# @TIER: STANDARD # @PURPOSE: Normalize intent entity value types from LLM output to route-compatible values. # @PRE: intent contains entities dict or missing entities. # @POST: Returned intent has numeric ids coerced where possible and string values stripped. @@ -1082,6 +1109,7 @@ _SAFE_OPS = {"show_capabilities", "get_task_status", "get_health_summary"} # [DEF:_confirmation_summary:Function] +# @TIER: STANDARD # @PURPOSE: Build human-readable confirmation prompt for an intent before execution. # @PRE: intent contains operation and entities fields. # @POST: Returns descriptive Russian-language text ending with confirmation prompt. @@ -1177,6 +1205,7 @@ async def _async_confirmation_summary(intent: Dict[str, Any], config_manager: Co # [DEF:_clarification_text_for_intent:Function] +# @TIER: STANDARD # @PURPOSE: Convert technical missing-parameter errors into user-facing clarification prompts. # @PRE: state was classified as needs_clarification for current intent/error combination. # @POST: Returned text is human-readable and actionable for target operation. @@ -1200,6 +1229,7 @@ def _clarification_text_for_intent(intent: Optional[Dict[str, Any]], detail_text # [DEF:_plan_intent_with_llm:Function] +# @TIER: STANDARD # @PURPOSE: Use active LLM provider to select best tool/operation from dynamic catalog. # @PRE: tools list contains allowed operations for current user. # @POST: Returns normalized intent dict when planning succeeds; otherwise None. @@ -1310,6 +1340,7 @@ async def _plan_intent_with_llm( # [DEF:_authorize_intent:Function] +# @TIER: STANDARD # @PURPOSE: Validate user permissions for parsed intent before confirmation/dispatch. # @PRE: intent.operation is present for known assistant command domains. # @POST: Returns if authorized; raises HTTPException(403) when denied. @@ -1321,6 +1352,7 @@ def _authorize_intent(intent: Dict[str, Any], current_user: User): # [DEF:_dispatch_intent:Function] +# @TIER: STANDARD # @PURPOSE: Execute parsed assistant intent via existing task/plugin/git services. # @PRE: intent operation is known and actor permissions are validated per operation. # @POST: Returns response text, optional task id, and UI actions for follow-up. @@ -1642,6 +1674,7 @@ async def _dispatch_intent( @router.post("/messages", response_model=AssistantMessageResponse) # [DEF:send_message:Function] +# @TIER: STANDARD # @PURPOSE: Parse assistant command, enforce safety gates, and dispatch executable intent. # @PRE: Authenticated user is available and message text is non-empty. # @POST: Response state is one of clarification/confirmation/started/success/denied/failed. @@ -1811,6 +1844,7 @@ async def send_message( @router.post("/confirmations/{confirmation_id}/confirm", response_model=AssistantMessageResponse) # [DEF:confirm_operation:Function] +# @TIER: STANDARD # @PURPOSE: Execute previously requested risky operation after explicit user confirmation. # @PRE: confirmation_id exists, belongs to current user, is pending, and not expired. # @POST: Confirmation state becomes consumed and operation result is persisted in history. @@ -1877,6 +1911,7 @@ async def confirm_operation( @router.post("/confirmations/{confirmation_id}/cancel", response_model=AssistantMessageResponse) # [DEF:cancel_operation:Function] +# @TIER: STANDARD # @PURPOSE: Cancel pending risky operation and mark confirmation token as cancelled. # @PRE: confirmation_id exists, belongs to current user, and is still pending. # @POST: Confirmation becomes cancelled and cannot be executed anymore. @@ -1933,6 +1968,7 @@ async def cancel_operation( # [DEF:list_conversations:Function] +# @TIER: STANDARD # @PURPOSE: Return paginated conversation list for current user with archived flag and last message preview. # @PRE: Authenticated user context and valid pagination params. # @POST: Conversations are grouped by conversation_id sorted by latest activity descending. @@ -2020,6 +2056,7 @@ async def list_conversations( # [DEF:delete_conversation:Function] +# @TIER: STANDARD # @PURPOSE: Soft-delete or hard-delete a conversation and clear its in-memory trace. # @PRE: conversation_id belongs to current_user. # @POST: Conversation records are removed from DB and CONVERSATIONS cache. diff --git a/backend/src/api/routes/dashboards.py b/backend/src/api/routes/dashboards.py index 51f5ea37..481e5d92 100644 --- a/backend/src/api/routes/dashboards.py +++ b/backend/src/api/routes/dashboards.py @@ -4,12 +4,17 @@ # @SEMANTICS: api, dashboards, resources, hub # @PURPOSE: API endpoints for the Dashboard Hub - listing dashboards with Git and task status # @LAYER: API -# @RELATION: DEPENDS_ON -> backend.src.dependencies -# @RELATION: DEPENDS_ON -> backend.src.services.resource_service -# @RELATION: DEPENDS_ON -> backend.src.core.superset_client +# @RELATION: DEPENDS_ON ->[backend.src.dependencies:Dependencies] +# @RELATION: DEPENDS_ON ->[backend.src.services.resource_service:ResourceService] +# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client:SupersetClient] # # @INVARIANT: All dashboard responses include git_status and last_task metadata # +# @PRE: Valid environment configurations exist in ConfigManager. +# @POST: Dashboard responses are projected into DashboardsResponse DTO. +# @SIDE_EFFECT: Performs external calls to Superset API and potentially Git providers. +# @DATA_CONTRACT: Input(env_id, filters) -> Output(DashboardsResponse) +# # @TEST_CONTRACT: DashboardsAPI -> { # required_fields: {env_id: string, page: integer, page_size: integer}, # optional_fields: {search: string}, @@ -61,6 +66,8 @@ from ...services.resource_service import ResourceService router = APIRouter(prefix="/api/dashboards", tags=["Dashboards"]) # [DEF:GitStatus:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for dashboard Git synchronization status. class GitStatus(BaseModel): branch: Optional[str] = None sync_status: Optional[str] = Field(None, pattern="^OK|DIFF|NO_REPO|ERROR$") @@ -69,6 +76,8 @@ class GitStatus(BaseModel): # [/DEF:GitStatus:DataClass] # [DEF:LastTask:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for the most recent background task associated with a dashboard. class LastTask(BaseModel): task_id: Optional[str] = None status: Optional[str] = Field( @@ -79,6 +88,8 @@ class LastTask(BaseModel): # [/DEF:LastTask:DataClass] # [DEF:DashboardItem:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO representing a single dashboard with projected metadata. class DashboardItem(BaseModel): id: int title: str @@ -93,6 +104,8 @@ class DashboardItem(BaseModel): # [/DEF:DashboardItem:DataClass] # [DEF:EffectiveProfileFilter:DataClass] +# @TIER: STANDARD +# @PURPOSE: Metadata about applied profile filters for UI context. class EffectiveProfileFilter(BaseModel): applied: bool source_page: Literal["dashboards_main", "other"] = "dashboards_main" @@ -104,6 +117,8 @@ class EffectiveProfileFilter(BaseModel): # [/DEF:EffectiveProfileFilter:DataClass] # [DEF:DashboardsResponse:DataClass] +# @TIER: STANDARD +# @PURPOSE: Envelope DTO for paginated dashboards list. class DashboardsResponse(BaseModel): dashboards: List[DashboardItem] total: int @@ -114,6 +129,8 @@ class DashboardsResponse(BaseModel): # [/DEF:DashboardsResponse:DataClass] # [DEF:DashboardChartItem:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for a chart linked to a dashboard. class DashboardChartItem(BaseModel): id: int title: str @@ -124,6 +141,8 @@ class DashboardChartItem(BaseModel): # [/DEF:DashboardChartItem:DataClass] # [DEF:DashboardDatasetItem:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for a dataset associated with a dashboard. class DashboardDatasetItem(BaseModel): id: int table_name: str @@ -134,6 +153,8 @@ class DashboardDatasetItem(BaseModel): # [/DEF:DashboardDatasetItem:DataClass] # [DEF:DashboardDetailResponse:DataClass] +# @TIER: STANDARD +# @PURPOSE: Detailed dashboard metadata including children. class DashboardDetailResponse(BaseModel): id: int title: str @@ -149,6 +170,8 @@ class DashboardDetailResponse(BaseModel): # [/DEF:DashboardDetailResponse:DataClass] # [DEF:DashboardTaskHistoryItem:DataClass] +# @TIER: STANDARD +# @PURPOSE: Individual history record entry. class DashboardTaskHistoryItem(BaseModel): id: str plugin_id: str @@ -161,12 +184,16 @@ class DashboardTaskHistoryItem(BaseModel): # [/DEF:DashboardTaskHistoryItem:DataClass] # [DEF:DashboardTaskHistoryResponse:DataClass] +# @TIER: STANDARD +# @PURPOSE: Collection DTO for task history. class DashboardTaskHistoryResponse(BaseModel): dashboard_id: int items: List[DashboardTaskHistoryItem] # [/DEF:DashboardTaskHistoryResponse:DataClass] # [DEF:DatabaseMapping:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for cross-environment database ID mapping. class DatabaseMapping(BaseModel): source_db: str target_db: str @@ -176,12 +203,15 @@ class DatabaseMapping(BaseModel): # [/DEF:DatabaseMapping:DataClass] # [DEF:DatabaseMappingsResponse:DataClass] +# @TIER: STANDARD +# @PURPOSE: Wrapper for database mappings. class DatabaseMappingsResponse(BaseModel): mappings: List[DatabaseMapping] # [/DEF:DatabaseMappingsResponse:DataClass] # [DEF:_find_dashboard_id_by_slug:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard numeric ID by slug using Superset list endpoint. # @PRE: `dashboard_slug` is non-empty. # @POST: Returns dashboard ID when found, otherwise None. @@ -209,6 +239,7 @@ def _find_dashboard_id_by_slug( # [DEF:_resolve_dashboard_id_from_ref:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard ID from slug-first reference with numeric fallback. # @PRE: `dashboard_ref` is provided in route path. # @POST: Returns a valid dashboard ID or raises HTTPException(404). @@ -233,6 +264,7 @@ def _resolve_dashboard_id_from_ref( # [DEF:_find_dashboard_id_by_slug_async:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard numeric ID by slug using async Superset list endpoint. # @PRE: dashboard_slug is non-empty. # @POST: Returns dashboard ID when found, otherwise None. @@ -260,6 +292,7 @@ async def _find_dashboard_id_by_slug_async( # [DEF:_resolve_dashboard_id_from_ref_async:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard ID from slug-first reference using async Superset client. # @PRE: dashboard_ref is provided in route path. # @POST: Returns valid dashboard ID or raises HTTPException(404). @@ -283,6 +316,7 @@ async def _resolve_dashboard_id_from_ref_async( # [DEF:_normalize_filter_values:Function] +# @TIER: STANDARD # @PURPOSE: Normalize query filter values to lower-cased non-empty tokens. # @PRE: values may be None or list of strings. # @POST: Returns trimmed normalized list preserving input order. @@ -299,6 +333,7 @@ def _normalize_filter_values(values: Optional[List[str]]) -> List[str]: # [DEF:_dashboard_git_filter_value:Function] +# @TIER: STANDARD # @PURPOSE: Build comparable git status token for dashboards filtering. # @PRE: dashboard payload may contain git_status or None. # @POST: Returns one of ok|diff|no_repo|error|pending. @@ -318,6 +353,7 @@ def _dashboard_git_filter_value(dashboard: Dict[str, Any]) -> str: # [/DEF:_dashboard_git_filter_value:Function] # [DEF:_normalize_actor_alias_token:Function] +# @TIER: STANDARD # @PURPOSE: Normalize actor alias token to comparable trim+lower text. # @PRE: value can be scalar/None. # @POST: Returns normalized token or None. @@ -328,6 +364,7 @@ def _normalize_actor_alias_token(value: Any) -> Optional[str]: # [DEF:_normalize_owner_display_token:Function] +# @TIER: STANDARD # @PURPOSE: Project owner payload value into stable display string for API response contracts. # @PRE: owner can be scalar, dict or None. # @POST: Returns trimmed non-empty owner display token or None. @@ -354,6 +391,7 @@ def _normalize_owner_display_token(owner: Any) -> Optional[str]: # [DEF:_normalize_dashboard_owner_values:Function] +# @TIER: STANDARD # @PURPOSE: Normalize dashboard owners payload to optional list of display strings. # @PRE: owners payload can be None, scalar, or list with mixed values. # @POST: Returns deduplicated owner labels preserving order, or None when absent. @@ -378,6 +416,7 @@ def _normalize_dashboard_owner_values(owners: Any) -> Optional[List[str]]: # [DEF:_project_dashboard_response_items:Function] +# @TIER: STANDARD # @PURPOSE: Project dashboard payloads to response-contract-safe shape. # @PRE: dashboards is a list of dict-like dashboard payloads. # @POST: Returned items satisfy DashboardItem owners=list[str]|None contract. @@ -394,6 +433,7 @@ def _project_dashboard_response_items(dashboards: List[Dict[str, Any]]) -> List[ # [DEF:_resolve_profile_actor_aliases:Function] +# @TIER: STANDARD # @PURPOSE: Resolve stable actor aliases for profile filtering without per-dashboard detail fan-out. # @PRE: bound username is available and env is valid. # @POST: Returns at least normalized username; may include Superset display-name alias. @@ -458,6 +498,7 @@ def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]: # [DEF:_matches_dashboard_actor_aliases:Function] +# @TIER: STANDARD # @PURPOSE: Apply profile actor matching against multiple aliases (username + optional display name). # @PRE: actor_aliases contains normalized non-empty tokens. # @POST: Returns True when any alias matches owners OR modified_by. @@ -479,6 +520,7 @@ def _matches_dashboard_actor_aliases( # [DEF:get_dashboards:Function] +# @TIER: STANDARD # @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status # @PRE: env_id must be a valid environment ID # @PRE: page must be >= 1 if provided @@ -491,7 +533,7 @@ def _matches_dashboard_actor_aliases( # @PARAM: page (Optional[int]) - Page number (default: 1) # @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) # @RETURN: DashboardsResponse - List of dashboards with status metadata -# @RELATION: CALLS -> ResourceService.get_dashboards_with_status +# @RELATION: CALLS ->[ResourceService:get_dashboards_with_status] @router.get("", response_model=DashboardsResponse) async def get_dashboards( env_id: str, @@ -781,6 +823,7 @@ async def get_dashboards( # [/DEF:get_dashboards:Function] # [DEF:get_database_mappings:Function] +# @TIER: STANDARD # @PURPOSE: Get database mapping suggestions between source and target environments # @PRE: User has permission plugin:migration:read # @PRE: source_env_id and target_env_id are valid environment IDs @@ -788,7 +831,7 @@ async def get_dashboards( # @PARAM: source_env_id (str) - Source environment ID # @PARAM: target_env_id (str) - Target environment ID # @RETURN: DatabaseMappingsResponse - List of suggested mappings -# @RELATION: CALLS -> MappingService.get_suggestions +# @RELATION: CALLS ->[MappingService:get_suggestions] @router.get("/db-mappings", response_model=DatabaseMappingsResponse) async def get_database_mappings( source_env_id: str, @@ -836,10 +879,11 @@ async def get_database_mappings( # [/DEF:get_database_mappings:Function] # [DEF:get_dashboard_detail:Function] +# @TIER: STANDARD # @PURPOSE: Fetch detailed dashboard info with related charts and datasets # @PRE: env_id must be valid and dashboard ref (slug or id) must exist # @POST: Returns dashboard detail payload for overview page -# @RELATION: CALLS -> SupersetClient.get_dashboard_detail +# @RELATION: CALLS ->[AsyncSupersetClient:get_dashboard_detail_async] @router.get("/{dashboard_ref}", response_model=DashboardDetailResponse) async def get_dashboard_detail( dashboard_ref: str, @@ -873,6 +917,7 @@ async def get_dashboard_detail( # [DEF:_task_matches_dashboard:Function] +# @TIER: STANDARD # @PURPOSE: Checks whether task params are tied to a specific dashboard and environment. # @PRE: task-like object exposes plugin_id and params fields. # @POST: Returns True only for supported task plugins tied to dashboard_id (+optional env_id). @@ -906,6 +951,7 @@ def _task_matches_dashboard(task: Any, dashboard_id: int, env_id: Optional[str]) # [DEF:get_dashboard_tasks_history:Function] +# @TIER: STANDARD # @PURPOSE: Returns history of backup and LLM validation tasks for a dashboard. # @PRE: dashboard ref (slug or id) is valid. # @POST: Response contains sorted task history (newest first). @@ -992,6 +1038,7 @@ async def get_dashboard_tasks_history( # [DEF:get_dashboard_thumbnail:Function] +# @TIER: STANDARD # @PURPOSE: Proxies Superset dashboard thumbnail with cache support. # @PRE: env_id must exist. # @POST: Returns image bytes or 202 when thumbnail is being prepared by Superset. @@ -1072,7 +1119,7 @@ async def get_dashboard_thumbnail( content_type = thumb_response.headers.get("Content-Type", "image/png") return Response(content=thumb_response.content, media_type=content_type) - except DashboardNotFoundError as e: + except DashboardNotFoundError as e: logger.error(f"[get_dashboard_thumbnail][Coherence:Failed] Dashboard not found for thumbnail: {e}") raise HTTPException(status_code=404, detail="Dashboard thumbnail not found") except HTTPException: @@ -1085,6 +1132,8 @@ async def get_dashboard_thumbnail( # [/DEF:get_dashboard_thumbnail:Function] # [DEF:MigrateRequest:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for dashboard migration requests. class MigrateRequest(BaseModel): source_env_id: str = Field(..., description="Source environment ID") target_env_id: str = Field(..., description="Target environment ID") @@ -1094,11 +1143,14 @@ class MigrateRequest(BaseModel): # [/DEF:MigrateRequest:DataClass] # [DEF:TaskResponse:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for async task ID return. class TaskResponse(BaseModel): task_id: str # [/DEF:TaskResponse:DataClass] # [DEF:migrate_dashboards:Function] +# @TIER: STANDARD # @PURPOSE: Trigger bulk migration of dashboards from source to target environment # @PRE: User has permission plugin:migration:execute # @PRE: source_env_id and target_env_id are valid environment IDs @@ -1107,8 +1159,8 @@ class TaskResponse(BaseModel): # @POST: Task is created and queued for execution # @PARAM: request (MigrateRequest) - Migration request with source, target, and dashboard IDs # @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES -> MigrationPlugin -# @RELATION: CALLS -> task_manager.create_task +# @RELATION: DISPATCHES ->[MigrationPlugin:execute] +# @RELATION: CALLS ->[task_manager:create_task] @router.post("/migrate", response_model=TaskResponse) async def migrate_dashboards( request: MigrateRequest, @@ -1159,6 +1211,8 @@ async def migrate_dashboards( # [/DEF:migrate_dashboards:Function] # [DEF:BackupRequest:DataClass] +# @TIER: STANDARD +# @PURPOSE: DTO for dashboard backup requests. class BackupRequest(BaseModel): env_id: str = Field(..., description="Environment ID") dashboard_ids: List[int] = Field(..., description="List of dashboard IDs to backup") @@ -1166,6 +1220,7 @@ class BackupRequest(BaseModel): # [/DEF:BackupRequest:DataClass] # [DEF:backup_dashboards:Function] +# @TIER: STANDARD # @PURPOSE: Trigger bulk backup of dashboards with optional cron schedule # @PRE: User has permission plugin:backup:execute # @PRE: env_id is a valid environment ID @@ -1175,8 +1230,8 @@ class BackupRequest(BaseModel): # @POST: If schedule is provided, a scheduled task is created # @PARAM: request (BackupRequest) - Backup request with environment and dashboard IDs # @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES -> BackupPlugin -# @RELATION: CALLS -> task_manager.create_task +# @RELATION: DISPATCHES ->[BackupPlugin:execute] +# @RELATION: CALLS ->[task_manager:create_task] @router.post("/backup", response_model=TaskResponse) async def backup_dashboards( request: BackupRequest, diff --git a/backend/src/api/routes/datasets.py b/backend/src/api/routes/datasets.py index 2695cded..4394a7d6 100644 --- a/backend/src/api/routes/datasets.py +++ b/backend/src/api/routes/datasets.py @@ -4,9 +4,9 @@ # @SEMANTICS: api, datasets, resources, hub # @PURPOSE: API endpoints for the Dataset Hub - listing datasets with mapping progress # @LAYER: API -# @RELATION: DEPENDS_ON -> backend.src.dependencies -# @RELATION: DEPENDS_ON -> backend.src.services.resource_service -# @RELATION: DEPENDS_ON -> backend.src.core.superset_client +# @RELATION: DEPENDS_ON ->[backend.src.dependencies] +# @RELATION: DEPENDS_ON ->[backend.src.services.resource_service] +# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client] # # @INVARIANT: All dataset responses include last_task metadata @@ -22,28 +22,39 @@ from ...core.superset_client import SupersetClient router = APIRouter(prefix="/api/datasets", tags=["Datasets"]) # [DEF:MappedFields:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: DTO for dataset mapping progress statistics class MappedFields(BaseModel): total: int mapped: int # [/DEF:MappedFields:DataClass] # [DEF:LastTask:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: DTO for the most recent task associated with a dataset class LastTask(BaseModel): task_id: Optional[str] = None status: Optional[str] = Field(None, pattern="^RUNNING|SUCCESS|ERROR|WAITING_INPUT$") # [/DEF:LastTask:DataClass] # [DEF:DatasetItem:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: Summary DTO for a dataset in the hub listing class DatasetItem(BaseModel): id: int table_name: str - schema: str + schema_name: str = Field(..., alias="schema") database: str mapped_fields: Optional[MappedFields] = None last_task: Optional[LastTask] = None + + class Config: + allow_population_by_field_name = True # [/DEF:DatasetItem:DataClass] # [DEF:LinkedDashboard:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: DTO for a dashboard linked to a dataset class LinkedDashboard(BaseModel): id: int title: str @@ -51,6 +62,8 @@ class LinkedDashboard(BaseModel): # [/DEF:LinkedDashboard:DataClass] # [DEF:DatasetColumn:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: DTO for a single dataset column's metadata class DatasetColumn(BaseModel): id: int name: str @@ -61,10 +74,12 @@ class DatasetColumn(BaseModel): # [/DEF:DatasetColumn:DataClass] # [DEF:DatasetDetailResponse:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: Detailed DTO for a dataset including columns and links class DatasetDetailResponse(BaseModel): id: int table_name: Optional[str] = None - schema: Optional[str] = None + schema_name: Optional[str] = Field(None, alias="schema") database: str description: Optional[str] = None columns: List[DatasetColumn] @@ -75,9 +90,14 @@ class DatasetDetailResponse(BaseModel): is_sqllab_view: bool = False created_on: Optional[str] = None changed_on: Optional[str] = None + + class Config: + allow_population_by_field_name = True # [/DEF:DatasetDetailResponse:DataClass] # [DEF:DatasetsResponse:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: Paginated response DTO for dataset listings class DatasetsResponse(BaseModel): datasets: List[DatasetItem] total: int @@ -87,18 +107,21 @@ class DatasetsResponse(BaseModel): # [/DEF:DatasetsResponse:DataClass] # [DEF:TaskResponse:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: Response DTO containing a task ID for tracking class TaskResponse(BaseModel): task_id: str # [/DEF:TaskResponse:DataClass] # [DEF:get_dataset_ids:Function] +# @TIER: STANDARD # @PURPOSE: Fetch list of all dataset IDs from a specific environment (without pagination) # @PRE: env_id must be a valid environment ID # @POST: Returns a list of all dataset IDs # @PARAM: env_id (str) - The environment ID to fetch datasets from # @PARAM: search (Optional[str]) - Filter by table name # @RETURN: List[int] - List of dataset IDs -# @RELATION: CALLS -> ResourceService.get_datasets_with_status +# @RELATION: CALLS ->[backend.src.services.resource_service.ResourceService:get_datasets_with_status] @router.get("/ids") async def get_dataset_ids( env_id: str, @@ -143,6 +166,7 @@ async def get_dataset_ids( # [/DEF:get_dataset_ids:Function] # [DEF:get_datasets:Function] +# @TIER: STANDARD # @PURPOSE: Fetch list of datasets from a specific environment with mapping progress # @PRE: env_id must be a valid environment ID # @PRE: page must be >= 1 if provided @@ -154,7 +178,7 @@ async def get_dataset_ids( # @PARAM: page (Optional[int]) - Page number (default: 1) # @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) # @RETURN: DatasetsResponse - List of datasets with status metadata -# @RELATION: CALLS -> ResourceService.get_datasets_with_status +# @RELATION: CALLS ->[backend.src.services.resource_service.ResourceService:get_datasets_with_status] @router.get("", response_model=DatasetsResponse) async def get_datasets( env_id: str, @@ -222,6 +246,8 @@ async def get_datasets( # [/DEF:get_datasets:Function] # [DEF:MapColumnsRequest:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: Request DTO for initiating column mapping class MapColumnsRequest(BaseModel): env_id: str = Field(..., description="Environment ID") dataset_ids: List[int] = Field(..., description="List of dataset IDs to map") @@ -231,6 +257,7 @@ class MapColumnsRequest(BaseModel): # [/DEF:MapColumnsRequest:DataClass] # [DEF:map_columns:Function] +# @TIER: STANDARD # @PURPOSE: Trigger bulk column mapping for datasets # @PRE: User has permission plugin:mapper:execute # @PRE: env_id is a valid environment ID @@ -239,8 +266,8 @@ class MapColumnsRequest(BaseModel): # @POST: Task is created and queued for execution # @PARAM: request (MapColumnsRequest) - Mapping request with environment and dataset IDs # @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES -> MapperPlugin -# @RELATION: CALLS -> task_manager.create_task +# @RELATION: DISPATCHES ->[backend.src.plugins.mapper.MapperPlugin] +# @RELATION: CALLS ->[backend.src.core.task_manager.manager.TaskManager:create_task] @router.post("/map-columns", response_model=TaskResponse) async def map_columns( request: MapColumnsRequest, @@ -292,6 +319,8 @@ async def map_columns( # [/DEF:map_columns:Function] # [DEF:GenerateDocsRequest:DataClass] +# @TIER: TRIVIAL +# @PURPOSE: Request DTO for initiating documentation generation class GenerateDocsRequest(BaseModel): env_id: str = Field(..., description="Environment ID") dataset_ids: List[int] = Field(..., description="List of dataset IDs to generate docs for") @@ -300,6 +329,7 @@ class GenerateDocsRequest(BaseModel): # [/DEF:GenerateDocsRequest:DataClass] # [DEF:generate_docs:Function] +# @TIER: STANDARD # @PURPOSE: Trigger bulk documentation generation for datasets # @PRE: User has permission plugin:llm_analysis:execute # @PRE: env_id is a valid environment ID @@ -308,8 +338,8 @@ class GenerateDocsRequest(BaseModel): # @POST: Task is created and queued for execution # @PARAM: request (GenerateDocsRequest) - Documentation generation request # @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES -> LLMAnalysisPlugin -# @RELATION: CALLS -> task_manager.create_task +# @RELATION: DISPATCHES ->[backend.src.plugins.llm_analysis.plugin.DocumentationPlugin] +# @RELATION: CALLS ->[backend.src.core.task_manager.manager.TaskManager:create_task] @router.post("/generate-docs", response_model=TaskResponse) async def generate_docs( request: GenerateDocsRequest, @@ -355,6 +385,7 @@ async def generate_docs( # [/DEF:generate_docs:Function] # [DEF:get_dataset_detail:Function] +# @TIER: STANDARD # @PURPOSE: Get detailed dataset information including columns and linked dashboards # @PRE: env_id is a valid environment ID # @PRE: dataset_id is a valid dataset ID @@ -362,7 +393,7 @@ async def generate_docs( # @PARAM: env_id (str) - The environment ID # @PARAM: dataset_id (int) - The dataset ID # @RETURN: DatasetDetailResponse - Detailed dataset information -# @RELATION: CALLS -> SupersetClient.get_dataset_detail +# @RELATION: CALLS ->[backend.src.core.superset_client.SupersetClient:get_dataset_detail] @router.get("/{dataset_id}", response_model=DatasetDetailResponse) async def get_dataset_detail( env_id: str, diff --git a/backend/src/api/routes/git.py b/backend/src/api/routes/git.py index bbfe5947..3962bd5e 100644 --- a/backend/src/api/routes/git.py +++ b/backend/src/api/routes/git.py @@ -4,9 +4,9 @@ # @SEMANTICS: git, routes, api, fastapi, repository, deployment # @PURPOSE: Provides FastAPI endpoints for Git integration operations. # @LAYER: API -# @RELATION: USES -> src.services.git_service.GitService -# @RELATION: USES -> src.api.routes.git_schemas -# @RELATION: USES -> src.models.git +# @RELATION: USES -> [backend.src.services.git_service.GitService] +# @RELATION: USES -> [backend.src.api.routes.git_schemas] +# @RELATION: USES -> [backend.src.models.git] # # @INVARIANT: All Git operations must be routed through GitService. @@ -48,6 +48,7 @@ MAX_REPOSITORY_STATUS_BATCH = 50 # [DEF:_build_no_repo_status_payload:Function] +# @TIER: TRIVIAL # @PURPOSE: Build a consistent status payload for dashboards without initialized repositories. # @PRE: None. # @POST: Returns a stable payload compatible with frontend repository status parsing. @@ -72,6 +73,7 @@ def _build_no_repo_status_payload() -> dict: # [DEF:_handle_unexpected_git_route_error:Function] +# @TIER: TRIVIAL # @PURPOSE: Convert unexpected route-level exceptions to stable 500 API responses. # @PRE: `error` is a non-HTTPException instance. # @POST: Raises HTTPException(500) with route-specific context. @@ -84,6 +86,7 @@ def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> Non # [DEF:_resolve_repository_status:Function] +# @TIER: STANDARD # @PURPOSE: Resolve repository status for one dashboard with graceful NO_REPO semantics. # @PRE: `dashboard_id` is a valid integer. # @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent. @@ -110,6 +113,7 @@ def _resolve_repository_status(dashboard_id: int) -> dict: # [DEF:_get_git_config_or_404:Function] +# @TIER: STANDARD # @PURPOSE: Resolve GitServerConfig by id or raise 404. # @PRE: db session is available. # @POST: Returns GitServerConfig model. @@ -122,6 +126,7 @@ def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig: # [DEF:_find_dashboard_id_by_slug:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard numeric ID by slug in a specific environment. # @PRE: dashboard_slug is non-empty. # @POST: Returns dashboard ID or None when not found. @@ -148,6 +153,7 @@ def _find_dashboard_id_by_slug( # [DEF:_resolve_dashboard_id_from_ref:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard ID from slug-or-id reference for Git routes. # @PRE: dashboard_ref is provided; env_id is required for slug values. # @POST: Returns numeric dashboard ID or raises HTTPException. @@ -182,6 +188,7 @@ def _resolve_dashboard_id_from_ref( # [DEF:_find_dashboard_id_by_slug_async:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes. # @PRE: dashboard_slug is non-empty. # @POST: Returns dashboard ID or None when not found. @@ -208,6 +215,7 @@ async def _find_dashboard_id_by_slug_async( # [DEF:_resolve_dashboard_id_from_ref_async:Function] +# @TIER: STANDARD # @PURPOSE: Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes. # @PRE: dashboard_ref is provided; env_id is required for slug values. # @POST: Returns numeric dashboard ID or raises HTTPException. @@ -246,6 +254,7 @@ async def _resolve_dashboard_id_from_ref_async( # [DEF:_resolve_repo_key_from_ref:Function] +# @TIER: STANDARD # @PURPOSE: Resolve repository folder key with slug-first strategy and deterministic fallback. # @PRE: dashboard_id is resolved and valid. # @POST: Returns safe key to be used in local repository path. @@ -278,6 +287,7 @@ def _resolve_repo_key_from_ref( # [DEF:_sanitize_optional_identity_value:Function] +# @TIER: TRIVIAL # @PURPOSE: Normalize optional identity value into trimmed string or None. # @PRE: value may be None or blank. # @POST: Returns sanitized value suitable for git identity configuration. @@ -291,6 +301,7 @@ def _sanitize_optional_identity_value(value: Optional[str]) -> Optional[str]: # [DEF:_resolve_current_user_git_identity:Function] +# @TIER: STANDARD # @PURPOSE: Resolve configured Git username/email from current user's profile preferences. # @PRE: `db` may be stubbed in tests; `current_user` may be absent for direct handler invocations. # @POST: Returns tuple(username, email) only when both values are configured. @@ -332,6 +343,7 @@ def _resolve_current_user_git_identity( # [DEF:_apply_git_identity_from_profile:Function] +# @TIER: STANDARD # @PURPOSE: Apply user-scoped Git identity to repository-local config before write/pull operations. # @PRE: dashboard_id is resolved; db/current_user may be missing in direct test invocation context. # @POST: git_service.configure_identity is called only when identity and method are available. @@ -355,6 +367,7 @@ def _apply_git_identity_from_profile( # [DEF:get_git_configs:Function] +# @TIER: STANDARD # @PURPOSE: List all configured Git servers. # @PRE: Database session `db` is available. # @POST: Returns a list of all GitServerConfig objects from the database. @@ -375,6 +388,7 @@ async def get_git_configs( # [/DEF:get_git_configs:Function] # [DEF:create_git_config:Function] +# @TIER: STANDARD # @PURPOSE: Register a new Git server configuration. # @PRE: `config` contains valid GitServerConfigCreate data. # @POST: A new GitServerConfig record is created in the database. @@ -396,6 +410,7 @@ async def create_git_config( # [/DEF:create_git_config:Function] # [DEF:update_git_config:Function] +# @TIER: STANDARD # @PURPOSE: Update an existing Git server configuration. # @PRE: `config_id` corresponds to an existing configuration. # @POST: The configuration record is updated in the database. @@ -430,6 +445,7 @@ async def update_git_config( # [/DEF:update_git_config:Function] # [DEF:delete_git_config:Function] +# @TIER: STANDARD # @PURPOSE: Remove a Git server configuration. # @PRE: `config_id` corresponds to an existing configuration. # @POST: The configuration record is removed from the database. @@ -451,6 +467,7 @@ async def delete_git_config( # [/DEF:delete_git_config:Function] # [DEF:test_git_config:Function] +# @TIER: STANDARD # @PURPOSE: Validate connection to a Git server using provided credentials. # @PRE: `config` contains provider, url, and pat. # @POST: Returns success if the connection is validated via GitService. @@ -482,6 +499,7 @@ async def test_git_config( # [DEF:list_gitea_repositories:Function] +# @TIER: STANDARD # @PURPOSE: List repositories in Gitea for a saved Gitea config. # @PRE: config_id exists and provider is GITEA. # @POST: Returns repositories visible to PAT user. @@ -512,6 +530,7 @@ async def list_gitea_repositories( # [DEF:create_gitea_repository:Function] +# @TIER: STANDARD # @PURPOSE: Create a repository in Gitea for a saved Gitea config. # @PRE: config_id exists and provider is GITEA. # @POST: Returns created repository payload. @@ -548,6 +567,7 @@ async def create_gitea_repository( # [DEF:create_remote_repository:Function] +# @TIER: STANDARD # @PURPOSE: Create repository on remote Git server using selected provider config. # @PRE: config_id exists and PAT has creation permissions. # @POST: Returns normalized remote repository payload. @@ -608,6 +628,7 @@ async def create_remote_repository( # [DEF:delete_gitea_repository:Function] +# @TIER: STANDARD # @PURPOSE: Delete repository in Gitea for a saved Gitea config. # @PRE: config_id exists and provider is GITEA. # @POST: Target repository is deleted on Gitea. @@ -633,6 +654,7 @@ async def delete_gitea_repository( # [/DEF:delete_gitea_repository:Function] # [DEF:init_repository:Function] +# @TIER: STANDARD # @PURPOSE: Link a dashboard to a Git repository and perform initial clone/init. # @PRE: `dashboard_ref` exists and `init_data` contains valid config_id and remote_url. # @POST: Repository is initialized on disk and a GitRepository record is saved in DB. @@ -690,6 +712,7 @@ async def init_repository( # [/DEF:init_repository:Function] # [DEF:get_repository_binding:Function] +# @TIER: STANDARD # @PURPOSE: Return repository binding with provider metadata for selected dashboard. # @PRE: `dashboard_ref` resolves to a valid dashboard and repository is initialized. # @POST: Returns dashboard repository binding and linked provider. @@ -724,6 +747,7 @@ async def get_repository_binding( # [/DEF:get_repository_binding:Function] # [DEF:delete_repository:Function] +# @TIER: STANDARD # @PURPOSE: Delete local repository workspace and DB binding for selected dashboard. # @PRE: `dashboard_ref` resolves to a valid dashboard. # @POST: Repository files and binding record are removed when present. @@ -748,6 +772,7 @@ async def delete_repository( # [/DEF:delete_repository:Function] # [DEF:get_branches:Function] +# @TIER: STANDARD # @PURPOSE: List all branches for a dashboard's repository. # @PRE: Repository for `dashboard_ref` is initialized. # @POST: Returns a list of branches from the local repository. @@ -771,6 +796,7 @@ async def get_branches( # [/DEF:get_branches:Function] # [DEF:create_branch:Function] +# @TIER: STANDARD # @PURPOSE: Create a new branch in the dashboard's repository. # @PRE: `dashboard_ref` repository exists and `branch_data` has name and from_branch. # @POST: A new branch is created in the local repository. @@ -799,6 +825,7 @@ async def create_branch( # [/DEF:create_branch:Function] # [DEF:checkout_branch:Function] +# @TIER: STANDARD # @PURPOSE: Switch the dashboard's repository to a specific branch. # @PRE: `dashboard_ref` repository exists and branch `checkout_data.name` exists. # @POST: The local repository HEAD is moved to the specified branch. @@ -824,6 +851,7 @@ async def checkout_branch( # [/DEF:checkout_branch:Function] # [DEF:commit_changes:Function] +# @TIER: STANDARD # @PURPOSE: Stage and commit changes in the dashboard's repository. # @PRE: `dashboard_ref` repository exists and `commit_data` has message and files. # @POST: Specified files are staged and a new commit is created. @@ -852,6 +880,7 @@ async def commit_changes( # [/DEF:commit_changes:Function] # [DEF:push_changes:Function] +# @TIER: STANDARD # @PURPOSE: Push local commits to the remote repository. # @PRE: `dashboard_ref` repository exists and has a remote configured. # @POST: Local commits are pushed to the remote repository. @@ -875,6 +904,7 @@ async def push_changes( # [/DEF:push_changes:Function] # [DEF:pull_changes:Function] +# @TIER: STANDARD # @PURPOSE: Pull changes from the remote repository. # @PRE: `dashboard_ref` repository exists and has a remote configured. # @POST: Remote changes are fetched and merged into the local branch. @@ -922,6 +952,7 @@ async def pull_changes( # [/DEF:pull_changes:Function] # [DEF:get_merge_status:Function] +# @TIER: STANDARD # @PURPOSE: Return unfinished-merge status for repository (web-only recovery support). # @PRE: `dashboard_ref` resolves to a valid dashboard repository. # @POST: Returns merge status payload. @@ -944,6 +975,7 @@ async def get_merge_status( # [DEF:get_merge_conflicts:Function] +# @TIER: STANDARD # @PURPOSE: Return conflicted files with mine/theirs previews for web conflict resolver. # @PRE: `dashboard_ref` resolves to a valid dashboard repository. # @POST: Returns conflict file list. @@ -966,6 +998,7 @@ async def get_merge_conflicts( # [DEF:resolve_merge_conflicts:Function] +# @TIER: STANDARD # @PURPOSE: Apply mine/theirs/manual conflict resolutions from WebUI and stage files. # @PRE: `dashboard_ref` resolves; request contains at least one resolution item. # @POST: Resolved files are staged in index. @@ -993,6 +1026,7 @@ async def resolve_merge_conflicts( # [DEF:abort_merge:Function] +# @TIER: STANDARD # @PURPOSE: Abort unfinished merge from WebUI flow. # @PRE: `dashboard_ref` resolves to repository. # @POST: Merge operation is aborted or reports no active merge. @@ -1015,6 +1049,7 @@ async def abort_merge( # [DEF:continue_merge:Function] +# @TIER: STANDARD # @PURPOSE: Finalize unfinished merge from WebUI flow. # @PRE: All conflicts are resolved and staged. # @POST: Merge commit is created. @@ -1038,6 +1073,7 @@ async def continue_merge( # [DEF:sync_dashboard:Function] +# @TIER: STANDARD # @PURPOSE: Sync dashboard state from Superset to Git using the GitPlugin. # @PRE: `dashboard_ref` is valid; GitPlugin is available. # @POST: Dashboard YAMLs are exported from Superset and committed to Git. @@ -1069,6 +1105,7 @@ async def sync_dashboard( # [DEF:promote_dashboard:Function] +# @TIER: STANDARD # @PURPOSE: Promote changes between branches via MR or direct merge. # @PRE: dashboard repository is initialized and Git config is valid. # @POST: Returns promotion result metadata. @@ -1171,6 +1208,7 @@ async def promote_dashboard( # [/DEF:promote_dashboard:Function] # [DEF:get_environments:Function] +# @TIER: STANDARD # @PURPOSE: List all deployment environments. # @PRE: Config manager is accessible. # @POST: Returns a list of DeploymentEnvironmentSchema objects. @@ -1193,6 +1231,7 @@ async def get_environments( # [/DEF:get_environments:Function] # [DEF:deploy_dashboard:Function] +# @TIER: STANDARD # @PURPOSE: Deploy dashboard from Git to a target environment. # @PRE: `dashboard_ref` and `deploy_data.environment_id` are valid. # @POST: Dashboard YAMLs are read from Git and imported into the target Superset. @@ -1223,6 +1262,7 @@ async def deploy_dashboard( # [/DEF:deploy_dashboard:Function] # [DEF:get_history:Function] +# @TIER: STANDARD # @PURPOSE: View commit history for a dashboard's repository. # @PRE: `dashboard_ref` repository exists. # @POST: Returns a list of recent commits from the repository. @@ -1248,6 +1288,7 @@ async def get_history( # [/DEF:get_history:Function] # [DEF:get_repository_status:Function] +# @TIER: STANDARD # @PURPOSE: Get current Git status for a dashboard repository. # @PRE: `dashboard_ref` resolves to a valid dashboard. # @POST: Returns repository status; if repo is not initialized, returns `NO_REPO` payload. @@ -1272,6 +1313,7 @@ async def get_repository_status( # [DEF:get_repository_status_batch:Function] +# @TIER: STANDARD # @PURPOSE: Get Git statuses for multiple dashboard repositories in one request. # @PRE: `request.dashboard_ids` is provided. # @POST: Returns `statuses` map where each key is dashboard ID and value is repository status payload. @@ -1315,6 +1357,7 @@ async def get_repository_status_batch( # [/DEF:get_repository_status_batch:Function] # [DEF:get_repository_diff:Function] +# @TIER: STANDARD # @PURPOSE: Get Git diff for a dashboard repository. # @PRE: `dashboard_ref` repository exists. # @POST: Returns the diff text for the specified file or all changes. @@ -1343,6 +1386,7 @@ async def get_repository_diff( # [/DEF:get_repository_diff:Function] # [DEF:generate_commit_message:Function] +# @TIER: STANDARD # @PURPOSE: Generate a suggested commit message using LLM. # @PRE: Repository for `dashboard_ref` is initialized. # @POST: Returns a suggested commit message string. diff --git a/backend/src/api/routes/migration.py b/backend/src/api/routes/migration.py index b32c2143..124082bc 100644 --- a/backend/src/api/routes/migration.py +++ b/backend/src/api/routes/migration.py @@ -3,14 +3,18 @@ # @SEMANTICS: api, migration, dashboards, sync, dry-run # @PURPOSE: HTTP contract layer for migration orchestration, settings, dry-run, and mapping sync endpoints. # @LAYER: Infra -# @RELATION: [DEPENDS_ON] ->[backend.src.dependencies] -# @RELATION: [DEPENDS_ON] ->[backend.src.core.database] -# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client] -# @RELATION: [DEPENDS_ON] ->[backend.src.core.migration.dry_run_orchestrator] -# @RELATION: [DEPENDS_ON] ->[backend.src.core.mapping_service] -# @RELATION: [DEPENDS_ON] ->[backend.src.models.dashboard] -# @RELATION: [DEPENDS_ON] ->[backend.src.models.mapping] +# @RELATION: DEPENDS_ON ->[backend.src.dependencies] +# @RELATION: DEPENDS_ON ->[backend.src.core.database] +# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client.SupersetClient] +# @RELATION: DEPENDS_ON ->[backend.src.core.migration.dry_run_orchestrator.MigrationDryRunService] +# @RELATION: DEPENDS_ON ->[backend.src.core.mapping_service.IdMappingService] +# @RELATION: DEPENDS_ON ->[backend.src.models.dashboard] +# @RELATION: DEPENDS_ON ->[backend.src.models.mapping] # @INVARIANT: Migration endpoints never execute with invalid environment references and always return explicit HTTP errors on guard failures. +# @PRE: Backend core services initialized and Database session available. +# @POST: Migration tasks are enqueued or dry-run results are computed and returned. +# @SIDE_EFFECT: Enqueues long-running tasks, potentially mutates ResourceMapping table, and performs remote Superset API calls. +# @DATA_CONTRACT: [DashboardSelection | QueryParams] -> [TaskResponse | DryRunResult | MappingSummary] # @TEST_CONTRACT: [DashboardSelection + configured envs] -> [task_id | dry-run result | sync summary] # @TEST_SCENARIO: [invalid_environment] -> [HTTP_400_or_404] # @TEST_SCENARIO: [valid_execution] -> [success_payload_with_required_fields] @@ -34,6 +38,7 @@ from ...models.mapping import ResourceMapping router = APIRouter(prefix="/api", tags=["migration"]) # [DEF:get_dashboards:Function] +# @TIER: STANDARD # @PURPOSE: Fetch dashboard metadata from a requested environment for migration selection UI. # @PRE: env_id is provided and exists in configured environments. # @POST: Returns List[DashboardMetadata] for the resolved environment; emits HTTP_404 when environment is absent. @@ -61,6 +66,7 @@ async def get_dashboards( # [/DEF:get_dashboards:Function] # [DEF:execute_migration:Function] +# @TIER: CRITICAL # @PURPOSE: Validate migration selection and enqueue asynchronous migration task execution. # @PRE: DashboardSelection payload is valid and both source/target environments exist. # @POST: Returns {"task_id": str, "message": str} when task creation succeeds; emits HTTP_400/HTTP_500 on failure. @@ -102,6 +108,7 @@ async def execute_migration( # [DEF:dry_run_migration:Function] +# @TIER: CRITICAL # @PURPOSE: Build pre-flight migration diff and risk summary without mutating target systems. # @PRE: DashboardSelection is valid, source and target environments exist, differ, and selected_ids is non-empty. # @POST: Returns deterministic dry-run payload; emits HTTP_400 for guard violations and HTTP_500 for orchestrator value errors. @@ -153,6 +160,7 @@ async def dry_run_migration( # [/DEF:dry_run_migration:Function] # [DEF:get_migration_settings:Function] +# @TIER: STANDARD # @PURPOSE: Read and return configured migration synchronization cron expression. # @PRE: Configuration store is available and requester has READ permission. # @POST: Returns {"cron": str} reflecting current persisted settings value. @@ -170,6 +178,7 @@ async def get_migration_settings( # [/DEF:get_migration_settings:Function] # [DEF:update_migration_settings:Function] +# @TIER: STANDARD # @PURPOSE: Validate and persist migration synchronization cron expression update. # @PRE: Payload includes "cron" key and requester has WRITE permission. # @POST: Returns {"cron": str, "status": "updated"} and persists updated cron value. @@ -195,6 +204,7 @@ async def update_migration_settings( # [/DEF:update_migration_settings:Function] # [DEF:get_resource_mappings:Function] +# @TIER: STANDARD # @PURPOSE: Fetch synchronized resource mappings with optional filters and pagination for migration mappings view. # @PRE: skip>=0, 1<=limit<=500, DB session is active, requester has READ permission. # @POST: Returns {"items": [...], "total": int} where items reflect applied filters and pagination. @@ -245,6 +255,7 @@ async def get_resource_mappings( # [/DEF:get_resource_mappings:Function] # [DEF:trigger_sync_now:Function] +# @TIER: STANDARD # @PURPOSE: Trigger immediate ID synchronization for every configured environment. # @PRE: At least one environment is configured and requester has EXECUTE permission. # @POST: Returns sync summary with synced/failed counts after attempting all environments. diff --git a/backend/src/api/routes/reports.py b/backend/src/api/routes/reports.py index 63c0553c..d3c174b8 100644 --- a/backend/src/api/routes/reports.py +++ b/backend/src/api/routes/reports.py @@ -3,9 +3,13 @@ # @SEMANTICS: api, reports, list, detail, pagination, filters # @PURPOSE: FastAPI router for unified task report list and detail retrieval endpoints. # @LAYER: UI (API) -# @RELATION: DEPENDS_ON -> backend.src.services.reports.report_service.ReportsService -# @RELATION: DEPENDS_ON -> backend.src.dependencies +# @RELATION: DEPENDS_ON -> [backend.src.services.reports.report_service.ReportsService] +# @RELATION: DEPENDS_ON -> [backend.src.dependencies] # @INVARIANT: Endpoints are read-only and do not trigger long-running tasks. +# @PRE: Reports service and dependencies are initialized. +# @POST: Router is configured and endpoints are ready for registration. +# @SIDE_EFFECT: None +# @DATA_CONTRACT: [ReportQuery] -> [ReportCollection | ReportDetailView] # [SECTION: IMPORTS] from datetime import datetime @@ -25,6 +29,7 @@ router = APIRouter(prefix="/api/reports", tags=["Reports"]) # [DEF:_parse_csv_enum_list:Function] +# @TIER: TRIVIAL # @PURPOSE: Parse comma-separated query value into enum list. # @PRE: raw may be None/empty or comma-separated values. # @POST: Returns enum list or raises HTTP 400 with deterministic machine-readable payload. @@ -59,6 +64,7 @@ def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List: # [DEF:list_reports:Function] +# @TIER: STANDARD # @PURPOSE: Return paginated unified reports list. # @PRE: authenticated/authorized request and validated query params. # @POST: returns {items,total,page,page_size,has_next,applied_filters}. @@ -125,6 +131,7 @@ async def list_reports( # [DEF:get_report_detail:Function] +# @TIER: STANDARD # @PURPOSE: Return one normalized report detail with diagnostics and next actions. # @PRE: authenticated/authorized request and existing report_id. # @POST: returns normalized detail envelope or 404 when report is not found. diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index 94c74bb9..6572f583 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -1,504 +1,527 @@ -# [DEF:SettingsRouter:Module] -# -# @SEMANTICS: settings, api, router, fastapi -# @PURPOSE: Provides API endpoints for managing application settings and Superset environments. -# @LAYER: UI (API) -# @RELATION: DEPENDS_ON -> ConfigManager -# @RELATION: DEPENDS_ON -> ConfigModels -# -# @INVARIANT: All settings changes must be persisted via ConfigManager. -# @PUBLIC_API: router - -# [SECTION: IMPORTS] -from fastapi import APIRouter, Depends, HTTPException -from typing import List -from pydantic import BaseModel -from ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig -from ...models.storage import StorageConfig -from ...dependencies import get_config_manager, has_permission -from ...core.config_manager import ConfigManager -from ...core.logger import logger, belief_scope -from ...core.superset_client import SupersetClient -from ...services.llm_prompt_templates import normalize_llm_settings -from ...models.llm import ValidationPolicy -from ...models.config import AppConfigRecord -from ...schemas.settings import ValidationPolicyCreate, ValidationPolicyUpdate, ValidationPolicyResponse -from ...core.database import get_db -from sqlalchemy.orm import Session -# [/SECTION] - -# [DEF:LoggingConfigResponse:Class] -# @PURPOSE: Response model for logging configuration with current task log level. -# @SEMANTICS: logging, config, response -class LoggingConfigResponse(BaseModel): - level: str - task_log_level: str - enable_belief_state: bool -# [/DEF:LoggingConfigResponse:Class] - -router = APIRouter() - - -# [DEF:_normalize_superset_env_url:Function] -# @PURPOSE: Canonicalize Superset environment URL to base host/path without trailing /api/v1. -# @PRE: raw_url can be empty. -# @POST: Returns normalized base URL. -def _normalize_superset_env_url(raw_url: str) -> str: - normalized = str(raw_url or "").strip().rstrip("/") - if normalized.lower().endswith("/api/v1"): - normalized = normalized[:-len("/api/v1")] - return normalized.rstrip("/") -# [/DEF:_normalize_superset_env_url:Function] - - -# [DEF:_validate_superset_connection_fast:Function] -# @PURPOSE: Run lightweight Superset connectivity validation without full pagination scan. -# @PRE: env contains valid URL and credentials. -# @POST: Raises on auth/API failures; returns None on success. -def _validate_superset_connection_fast(env: Environment) -> None: - client = SupersetClient(env) - # 1) Explicit auth check - client.authenticate() - # 2) Single lightweight API call to ensure read access - client.get_dashboards_page( - query={ - "page": 0, - "page_size": 1, - "columns": ["id"], - } - ) -# [/DEF:_validate_superset_connection_fast:Function] - -# [DEF:get_settings:Function] -# @PURPOSE: Retrieves all application settings. -# @PRE: Config manager is available. -# @POST: Returns masked AppConfig. -# @RETURN: AppConfig - The current configuration. -@router.get("", response_model=AppConfig) -async def get_settings( - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "READ")) -): - with belief_scope("get_settings"): - logger.info("[get_settings][Entry] Fetching all settings") - config = config_manager.get_config().copy(deep=True) - config.settings.llm = normalize_llm_settings(config.settings.llm) - # Mask passwords - for env in config.environments: - if env.password: - env.password = "********" - return config -# [/DEF:get_settings:Function] - -# [DEF:update_global_settings:Function] -# @PURPOSE: Updates global application settings. -# @PRE: New settings are provided. -# @POST: Global settings are updated. -# @PARAM: settings (GlobalSettings) - The new global settings. -# @RETURN: GlobalSettings - The updated settings. -@router.patch("/global", response_model=GlobalSettings) -async def update_global_settings( - settings: GlobalSettings, - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("update_global_settings"): - logger.info("[update_global_settings][Entry] Updating global settings") - - config_manager.update_global_settings(settings) - return settings -# [/DEF:update_global_settings:Function] - -# [DEF:get_storage_settings:Function] -# @PURPOSE: Retrieves storage-specific settings. -# @RETURN: StorageConfig - The storage configuration. -@router.get("/storage", response_model=StorageConfig) -async def get_storage_settings( - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "READ")) -): - with belief_scope("get_storage_settings"): - return config_manager.get_config().settings.storage -# [/DEF:get_storage_settings:Function] - -# [DEF:update_storage_settings:Function] -# @PURPOSE: Updates storage-specific settings. -# @PARAM: storage (StorageConfig) - The new storage settings. -# @POST: Storage settings are updated and saved. -# @RETURN: StorageConfig - The updated storage settings. -@router.put("/storage", response_model=StorageConfig) -async def update_storage_settings( - storage: StorageConfig, - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("update_storage_settings"): - is_valid, message = config_manager.validate_path(storage.root_path) - if not is_valid: - raise HTTPException(status_code=400, detail=message) - - settings = config_manager.get_config().settings - settings.storage = storage - config_manager.update_global_settings(settings) - return config_manager.get_config().settings.storage -# [/DEF:update_storage_settings:Function] - -# [DEF:get_environments:Function] -# @PURPOSE: Lists all configured Superset environments. -# @PRE: Config manager is available. -# @POST: Returns list of environments. -# @RETURN: List[Environment] - List of environments. -@router.get("/environments", response_model=List[Environment]) -async def get_environments( - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "READ")) -): - with belief_scope("get_environments"): - logger.info("[get_environments][Entry] Fetching environments") - environments = config_manager.get_environments() - return [ - env.copy(update={"url": _normalize_superset_env_url(env.url)}) - for env in environments - ] -# [/DEF:get_environments:Function] - -# [DEF:add_environment:Function] -# @PURPOSE: Adds a new Superset environment. -# @PRE: Environment data is valid and reachable. -# @POST: Environment is added to config. -# @PARAM: env (Environment) - The environment to add. -# @RETURN: Environment - The added environment. -@router.post("/environments", response_model=Environment) -async def add_environment( - env: Environment, - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("add_environment"): - logger.info(f"[add_environment][Entry] Adding environment {env.id}") - env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) - - # Validate connection before adding (fast path) - try: - _validate_superset_connection_fast(env) - except Exception as e: - logger.error(f"[add_environment][Coherence:Failed] Connection validation failed: {e}") - raise HTTPException(status_code=400, detail=f"Connection validation failed: {e}") - - config_manager.add_environment(env) - return env -# [/DEF:add_environment:Function] - -# [DEF:update_environment:Function] -# @PURPOSE: Updates an existing Superset environment. -# @PRE: ID and valid environment data are provided. -# @POST: Environment is updated in config. -# @PARAM: id (str) - The ID of the environment to update. -# @PARAM: env (Environment) - The updated environment data. -# @RETURN: Environment - The updated environment. -@router.put("/environments/{id}", response_model=Environment) -async def update_environment( - id: str, - env: Environment, - config_manager: ConfigManager = Depends(get_config_manager) -): - with belief_scope("update_environment"): - logger.info(f"[update_environment][Entry] Updating environment {id}") - - env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) - - # If password is masked, we need the real one for validation - env_to_validate = env.copy(deep=True) - if env_to_validate.password == "********": - old_env = next((e for e in config_manager.get_environments() if e.id == id), None) - if old_env: - env_to_validate.password = old_env.password - - # Validate connection before updating (fast path) - try: - _validate_superset_connection_fast(env_to_validate) - except Exception as e: - logger.error(f"[update_environment][Coherence:Failed] Connection validation failed: {e}") - raise HTTPException(status_code=400, detail=f"Connection validation failed: {e}") - - if config_manager.update_environment(id, env): - return env - raise HTTPException(status_code=404, detail=f"Environment {id} not found") -# [/DEF:update_environment:Function] - -# [DEF:delete_environment:Function] -# @PURPOSE: Deletes a Superset environment. -# @PRE: ID is provided. -# @POST: Environment is removed from config. -# @PARAM: id (str) - The ID of the environment to delete. -@router.delete("/environments/{id}") -async def delete_environment( - id: str, - config_manager: ConfigManager = Depends(get_config_manager) -): - with belief_scope("delete_environment"): - logger.info(f"[delete_environment][Entry] Deleting environment {id}") - config_manager.delete_environment(id) - return {"message": f"Environment {id} deleted"} -# [/DEF:delete_environment:Function] - -# [DEF:test_environment_connection:Function] -# @PURPOSE: Tests the connection to a Superset environment. -# @PRE: ID is provided. -# @POST: Returns success or error status. -# @PARAM: id (str) - The ID of the environment to test. -# @RETURN: dict - Success message or error. -@router.post("/environments/{id}/test") -async def test_environment_connection( - id: str, - config_manager: ConfigManager = Depends(get_config_manager) -): - with belief_scope("test_environment_connection"): - logger.info(f"[test_environment_connection][Entry] Testing environment {id}") - - # Find environment - env = next((e for e in config_manager.get_environments() if e.id == id), None) - if not env: - raise HTTPException(status_code=404, detail=f"Environment {id} not found") - - try: - _validate_superset_connection_fast(env) - - logger.info(f"[test_environment_connection][Coherence:OK] Connection successful for {id}") - return {"status": "success", "message": "Connection successful"} - except Exception as e: - logger.error(f"[test_environment_connection][Coherence:Failed] Connection failed for {id}: {e}") - return {"status": "error", "message": str(e)} -# [/DEF:test_environment_connection:Function] - -# [DEF:get_logging_config:Function] -# @PURPOSE: Retrieves current logging configuration. -# @PRE: Config manager is available. -# @POST: Returns logging configuration. -# @RETURN: LoggingConfigResponse - The current logging config. -@router.get("/logging", response_model=LoggingConfigResponse) -async def get_logging_config( - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "READ")) -): - with belief_scope("get_logging_config"): - logging_config = config_manager.get_config().settings.logging - return LoggingConfigResponse( - level=logging_config.level, - task_log_level=logging_config.task_log_level, - enable_belief_state=logging_config.enable_belief_state - ) -# [/DEF:get_logging_config:Function] - -# [DEF:update_logging_config:Function] -# @PURPOSE: Updates logging configuration. -# @PRE: New logging config is provided. -# @POST: Logging configuration is updated and saved. -# @PARAM: config (LoggingConfig) - The new logging configuration. -# @RETURN: LoggingConfigResponse - The updated logging config. -@router.patch("/logging", response_model=LoggingConfigResponse) -async def update_logging_config( - config: LoggingConfig, - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("update_logging_config"): - logger.info(f"[update_logging_config][Entry] Updating logging config: level={config.level}, task_log_level={config.task_log_level}") - - # Get current settings and update logging config - settings = config_manager.get_config().settings - settings.logging = config - config_manager.update_global_settings(settings) - - return LoggingConfigResponse( - level=config.level, - task_log_level=config.task_log_level, - enable_belief_state=config.enable_belief_state - ) -# [/DEF:update_logging_config:Function] - -# [DEF:ConsolidatedSettingsResponse:Class] -class ConsolidatedSettingsResponse(BaseModel): - environments: List[dict] - connections: List[dict] - llm: dict - llm_providers: List[dict] - logging: dict - storage: dict - notifications: dict = {} -# [/DEF:ConsolidatedSettingsResponse:Class] - -# [DEF:get_consolidated_settings:Function] -# @PURPOSE: Retrieves all settings categories in a single call -# @PRE: Config manager is available. -# @POST: Returns all consolidated settings. -# @RETURN: ConsolidatedSettingsResponse - All settings categories. -@router.get("/consolidated", response_model=ConsolidatedSettingsResponse) -async def get_consolidated_settings( - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "READ")) -): - with belief_scope("get_consolidated_settings"): - logger.info("[get_consolidated_settings][Entry] Fetching all consolidated settings") - - config = config_manager.get_config() - - from ...services.llm_provider import LLMProviderService - from ...core.database import SessionLocal - db = SessionLocal() - notifications_payload = {} - try: - llm_service = LLMProviderService(db) - providers = llm_service.get_all_providers() - llm_providers_list = [ - { - "id": p.id, - "provider_type": p.provider_type, - "name": p.name, - "base_url": p.base_url, - "api_key": "********", - "default_model": p.default_model, - "is_active": p.is_active - } for p in providers - ] - - config_record = db.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first() - if config_record and isinstance(config_record.payload, dict): - notifications_payload = config_record.payload.get("notifications", {}) or {} - finally: - db.close() - - normalized_llm = normalize_llm_settings(config.settings.llm) - - return ConsolidatedSettingsResponse( - environments=[env.dict() for env in config.environments], - connections=config.settings.connections, - llm=normalized_llm, - llm_providers=llm_providers_list, - logging=config.settings.logging.dict(), - storage=config.settings.storage.dict(), - notifications=notifications_payload - ) -# [/DEF:get_consolidated_settings:Function] - -# [DEF:update_consolidated_settings:Function] -# @PURPOSE: Bulk update application settings from the consolidated view. -# @PRE: User has admin permissions, config is valid. -# @POST: Settings are updated and saved via ConfigManager. -@router.patch("/consolidated") -async def update_consolidated_settings( - settings_patch: dict, - config_manager: ConfigManager = Depends(get_config_manager), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("update_consolidated_settings"): - logger.info("[update_consolidated_settings][Entry] Applying consolidated settings patch") - - current_config = config_manager.get_config() - current_settings = current_config.settings - - # Update connections if provided - if "connections" in settings_patch: - current_settings.connections = settings_patch["connections"] - - # Update LLM if provided - if "llm" in settings_patch: - current_settings.llm = normalize_llm_settings(settings_patch["llm"]) - - # Update Logging if provided - if "logging" in settings_patch: - current_settings.logging = LoggingConfig(**settings_patch["logging"]) - - # Update Storage if provided - if "storage" in settings_patch: - new_storage = StorageConfig(**settings_patch["storage"]) - is_valid, message = config_manager.validate_path(new_storage.root_path) - if not is_valid: - raise HTTPException(status_code=400, detail=message) - current_settings.storage = new_storage - - if "notifications" in settings_patch: - payload = config_manager.get_payload() - payload["notifications"] = settings_patch["notifications"] - config_manager.save_config(payload) - - config_manager.update_global_settings(current_settings) - return {"status": "success", "message": "Settings updated"} -# [/DEF:update_consolidated_settings:Function] - -# [DEF:get_validation_policies:Function] -# @PURPOSE: Lists all validation policies. -# @RETURN: List[ValidationPolicyResponse] - List of policies. -@router.get("/automation/policies", response_model=List[ValidationPolicyResponse]) -async def get_validation_policies( - db: Session = Depends(get_db), - _ = Depends(has_permission("admin:settings", "READ")) -): - with belief_scope("get_validation_policies"): - return db.query(ValidationPolicy).all() -# [/DEF:get_validation_policies:Function] - -# [DEF:create_validation_policy:Function] -# @PURPOSE: Creates a new validation policy. -# @PARAM: policy (ValidationPolicyCreate) - The policy data. -# @RETURN: ValidationPolicyResponse - The created policy. -@router.post("/automation/policies", response_model=ValidationPolicyResponse) -async def create_validation_policy( - policy: ValidationPolicyCreate, - db: Session = Depends(get_db), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("create_validation_policy"): - db_policy = ValidationPolicy(**policy.dict()) - db.add(db_policy) - db.commit() - db.refresh(db_policy) - return db_policy -# [/DEF:create_validation_policy:Function] - -# [DEF:update_validation_policy:Function] -# @PURPOSE: Updates an existing validation policy. -# @PARAM: id (str) - The ID of the policy to update. -# @PARAM: policy (ValidationPolicyUpdate) - The updated policy data. -# @RETURN: ValidationPolicyResponse - The updated policy. -@router.patch("/automation/policies/{id}", response_model=ValidationPolicyResponse) -async def update_validation_policy( - id: str, - policy: ValidationPolicyUpdate, - db: Session = Depends(get_db), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("update_validation_policy"): - db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first() - if not db_policy: - raise HTTPException(status_code=404, detail="Policy not found") - - update_data = policy.dict(exclude_unset=True) - for key, value in update_data.items(): - setattr(db_policy, key, value) - - db.commit() - db.refresh(db_policy) - return db_policy -# [/DEF:update_validation_policy:Function] - -# [DEF:delete_validation_policy:Function] -# @PURPOSE: Deletes a validation policy. -# @PARAM: id (str) - The ID of the policy to delete. -@router.delete("/automation/policies/{id}") -async def delete_validation_policy( - id: str, - db: Session = Depends(get_db), - _ = Depends(has_permission("admin:settings", "WRITE")) -): - with belief_scope("delete_validation_policy"): - db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first() - if not db_policy: - raise HTTPException(status_code=404, detail="Policy not found") - - db.delete(db_policy) - db.commit() - return {"message": "Policy deleted"} -# [/DEF:delete_validation_policy:Function] - -# [/DEF:SettingsRouter:Module] +# [DEF:SettingsRouter:Module] +# +# @TIER: STANDARD +# @SEMANTICS: settings, api, router, fastapi +# @PURPOSE: Provides API endpoints for managing application settings and Superset environments. +# @LAYER: UI (API) +# @RELATION: DEPENDS_ON -> [backend.src.core.config_manager.ConfigManager] +# @RELATION: DEPENDS_ON -> [backend.src.core.config_models] +# +# @INVARIANT: All settings changes must be persisted via ConfigManager. +# @PUBLIC_API: router + +# [SECTION: IMPORTS] +from fastapi import APIRouter, Depends, HTTPException +from typing import List +from pydantic import BaseModel +from ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig +from ...models.storage import StorageConfig +from ...dependencies import get_config_manager, has_permission +from ...core.config_manager import ConfigManager +from ...core.logger import logger, belief_scope +from ...core.superset_client import SupersetClient +from ...services.llm_prompt_templates import normalize_llm_settings +from ...models.llm import ValidationPolicy +from ...models.config import AppConfigRecord +from ...schemas.settings import ValidationPolicyCreate, ValidationPolicyUpdate, ValidationPolicyResponse +from ...core.database import get_db +from sqlalchemy.orm import Session +# [/SECTION] + +# [DEF:LoggingConfigResponse:Class] +# @TIER: TRIVIAL +# @PURPOSE: Response model for logging configuration with current task log level. +# @SEMANTICS: logging, config, response +class LoggingConfigResponse(BaseModel): + level: str + task_log_level: str + enable_belief_state: bool +# [/DEF:LoggingConfigResponse:Class] + +router = APIRouter() + + +# [DEF:_normalize_superset_env_url:Function] +# @TIER: TRIVIAL +# @PURPOSE: Canonicalize Superset environment URL to base host/path without trailing /api/v1. +# @PRE: raw_url can be empty. +# @POST: Returns normalized base URL. +def _normalize_superset_env_url(raw_url: str) -> str: + normalized = str(raw_url or "").strip().rstrip("/") + if normalized.lower().endswith("/api/v1"): + normalized = normalized[:-len("/api/v1")] + return normalized.rstrip("/") +# [/DEF:_normalize_superset_env_url:Function] + + +# [DEF:_validate_superset_connection_fast:Function] +# @TIER: STANDARD +# @PURPOSE: Run lightweight Superset connectivity validation without full pagination scan. +# @PRE: env contains valid URL and credentials. +# @POST: Raises on auth/API failures; returns None on success. +def _validate_superset_connection_fast(env: Environment) -> None: + client = SupersetClient(env) + # 1) Explicit auth check + client.authenticate() + # 2) Single lightweight API call to ensure read access + client.get_dashboards_page( + query={ + "page": 0, + "page_size": 1, + "columns": ["id"], + } + ) +# [/DEF:_validate_superset_connection_fast:Function] + +# [DEF:get_settings:Function] +# @TIER: STANDARD +# @PURPOSE: Retrieves all application settings. +# @PRE: Config manager is available. +# @POST: Returns masked AppConfig. +# @RETURN: AppConfig - The current configuration. +@router.get("", response_model=AppConfig) +async def get_settings( + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "READ")) +): + with belief_scope("get_settings"): + logger.info("[get_settings][Entry] Fetching all settings") + config = config_manager.get_config().copy(deep=True) + config.settings.llm = normalize_llm_settings(config.settings.llm) + # Mask passwords + for env in config.environments: + if env.password: + env.password = "********" + return config +# [/DEF:get_settings:Function] + +# [DEF:update_global_settings:Function] +# @TIER: STANDARD +# @PURPOSE: Updates global application settings. +# @PRE: New settings are provided. +# @POST: Global settings are updated. +# @PARAM: settings (GlobalSettings) - The new global settings. +# @RETURN: GlobalSettings - The updated settings. +@router.patch("/global", response_model=GlobalSettings) +async def update_global_settings( + settings: GlobalSettings, + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("update_global_settings"): + logger.info("[update_global_settings][Entry] Updating global settings") + + config_manager.update_global_settings(settings) + return settings +# [/DEF:update_global_settings:Function] + +# [DEF:get_storage_settings:Function] +# @TIER: STANDARD +# @PURPOSE: Retrieves storage-specific settings. +# @RETURN: StorageConfig - The storage configuration. +@router.get("/storage", response_model=StorageConfig) +async def get_storage_settings( + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "READ")) +): + with belief_scope("get_storage_settings"): + return config_manager.get_config().settings.storage +# [/DEF:get_storage_settings:Function] + +# [DEF:update_storage_settings:Function] +# @TIER: STANDARD +# @PURPOSE: Updates storage-specific settings. +# @PARAM: storage (StorageConfig) - The new storage settings. +# @POST: Storage settings are updated and saved. +# @RETURN: StorageConfig - The updated storage settings. +@router.put("/storage", response_model=StorageConfig) +async def update_storage_settings( + storage: StorageConfig, + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("update_storage_settings"): + is_valid, message = config_manager.validate_path(storage.root_path) + if not is_valid: + raise HTTPException(status_code=400, detail=message) + + settings = config_manager.get_config().settings + settings.storage = storage + config_manager.update_global_settings(settings) + return config_manager.get_config().settings.storage +# [/DEF:update_storage_settings:Function] + +# [DEF:get_environments:Function] +# @TIER: STANDARD +# @PURPOSE: Lists all configured Superset environments. +# @PRE: Config manager is available. +# @POST: Returns list of environments. +# @RETURN: List[Environment] - List of environments. +@router.get("/environments", response_model=List[Environment]) +async def get_environments( + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "READ")) +): + with belief_scope("get_environments"): + logger.info("[get_environments][Entry] Fetching environments") + environments = config_manager.get_environments() + return [ + env.copy(update={"url": _normalize_superset_env_url(env.url)}) + for env in environments + ] +# [/DEF:get_environments:Function] + +# [DEF:add_environment:Function] +# @TIER: STANDARD +# @PURPOSE: Adds a new Superset environment. +# @PRE: Environment data is valid and reachable. +# @POST: Environment is added to config. +# @PARAM: env (Environment) - The environment to add. +# @RETURN: Environment - The added environment. +@router.post("/environments", response_model=Environment) +async def add_environment( + env: Environment, + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("add_environment"): + logger.info(f"[add_environment][Entry] Adding environment {env.id}") + env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) + + # Validate connection before adding (fast path) + try: + _validate_superset_connection_fast(env) + except Exception as e: + logger.error(f"[add_environment][Coherence:Failed] Connection validation failed: {e}") + raise HTTPException(status_code=400, detail=f"Connection validation failed: {e}") + + config_manager.add_environment(env) + return env +# [/DEF:add_environment:Function] + +# [DEF:update_environment:Function] +# @TIER: STANDARD +# @PURPOSE: Updates an existing Superset environment. +# @PRE: ID and valid environment data are provided. +# @POST: Environment is updated in config. +# @PARAM: id (str) - The ID of the environment to update. +# @PARAM: env (Environment) - The updated environment data. +# @RETURN: Environment - The updated environment. +@router.put("/environments/{id}", response_model=Environment) +async def update_environment( + id: str, + env: Environment, + config_manager: ConfigManager = Depends(get_config_manager) +): + with belief_scope("update_environment"): + logger.info(f"[update_environment][Entry] Updating environment {id}") + + env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) + + # If password is masked, we need the real one for validation + env_to_validate = env.copy(deep=True) + if env_to_validate.password == "********": + old_env = next((e for e in config_manager.get_environments() if e.id == id), None) + if old_env: + env_to_validate.password = old_env.password + + # Validate connection before updating (fast path) + try: + _validate_superset_connection_fast(env_to_validate) + except Exception as e: + logger.error(f"[update_environment][Coherence:Failed] Connection validation failed: {e}") + raise HTTPException(status_code=400, detail=f"Connection validation failed: {e}") + + if config_manager.update_environment(id, env): + return env + raise HTTPException(status_code=404, detail=f"Environment {id} not found") +# [/DEF:update_environment:Function] + +# [DEF:delete_environment:Function] +# @TIER: STANDARD +# @PURPOSE: Deletes a Superset environment. +# @PRE: ID is provided. +# @POST: Environment is removed from config. +# @PARAM: id (str) - The ID of the environment to delete. +@router.delete("/environments/{id}") +async def delete_environment( + id: str, + config_manager: ConfigManager = Depends(get_config_manager) +): + with belief_scope("delete_environment"): + logger.info(f"[delete_environment][Entry] Deleting environment {id}") + config_manager.delete_environment(id) + return {"message": f"Environment {id} deleted"} +# [/DEF:delete_environment:Function] + +# [DEF:test_environment_connection:Function] +# @TIER: STANDARD +# @PURPOSE: Tests the connection to a Superset environment. +# @PRE: ID is provided. +# @POST: Returns success or error status. +# @PARAM: id (str) - The ID of the environment to test. +# @RETURN: dict - Success message or error. +@router.post("/environments/{id}/test") +async def test_environment_connection( + id: str, + config_manager: ConfigManager = Depends(get_config_manager) +): + with belief_scope("test_environment_connection"): + logger.info(f"[test_environment_connection][Entry] Testing environment {id}") + + # Find environment + env = next((e for e in config_manager.get_environments() if e.id == id), None) + if not env: + raise HTTPException(status_code=404, detail=f"Environment {id} not found") + + try: + _validate_superset_connection_fast(env) + + logger.info(f"[test_environment_connection][Coherence:OK] Connection successful for {id}") + return {"status": "success", "message": "Connection successful"} + except Exception as e: + logger.error(f"[test_environment_connection][Coherence:Failed] Connection failed for {id}: {e}") + return {"status": "error", "message": str(e)} +# [/DEF:test_environment_connection:Function] + +# [DEF:get_logging_config:Function] +# @TIER: STANDARD +# @PURPOSE: Retrieves current logging configuration. +# @PRE: Config manager is available. +# @POST: Returns logging configuration. +# @RETURN: LoggingConfigResponse - The current logging config. +@router.get("/logging", response_model=LoggingConfigResponse) +async def get_logging_config( + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "READ")) +): + with belief_scope("get_logging_config"): + logging_config = config_manager.get_config().settings.logging + return LoggingConfigResponse( + level=logging_config.level, + task_log_level=logging_config.task_log_level, + enable_belief_state=logging_config.enable_belief_state + ) +# [/DEF:get_logging_config:Function] + +# [DEF:update_logging_config:Function] +# @TIER: STANDARD +# @PURPOSE: Updates logging configuration. +# @PRE: New logging config is provided. +# @POST: Logging configuration is updated and saved. +# @PARAM: config (LoggingConfig) - The new logging configuration. +# @RETURN: LoggingConfigResponse - The updated logging config. +@router.patch("/logging", response_model=LoggingConfigResponse) +async def update_logging_config( + config: LoggingConfig, + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("update_logging_config"): + logger.info(f"[update_logging_config][Entry] Updating logging config: level={config.level}, task_log_level={config.task_log_level}") + + # Get current settings and update logging config + settings = config_manager.get_config().settings + settings.logging = config + config_manager.update_global_settings(settings) + + return LoggingConfigResponse( + level=config.level, + task_log_level=config.task_log_level, + enable_belief_state=config.enable_belief_state + ) +# [/DEF:update_logging_config:Function] + +# [DEF:ConsolidatedSettingsResponse:Class] +# @TIER: TRIVIAL +# @PURPOSE: Response model for consolidated application settings. +class ConsolidatedSettingsResponse(BaseModel): + environments: List[dict] + connections: List[dict] + llm: dict + llm_providers: List[dict] + logging: dict + storage: dict + notifications: dict = {} +# [/DEF:ConsolidatedSettingsResponse:Class] + +# [DEF:get_consolidated_settings:Function] +# @TIER: STANDARD +# @PURPOSE: Retrieves all settings categories in a single call +# @PRE: Config manager is available. +# @POST: Returns all consolidated settings. +# @RETURN: ConsolidatedSettingsResponse - All settings categories. +@router.get("/consolidated", response_model=ConsolidatedSettingsResponse) +async def get_consolidated_settings( + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "READ")) +): + with belief_scope("get_consolidated_settings"): + logger.info("[get_consolidated_settings][Entry] Fetching all consolidated settings") + + config = config_manager.get_config() + + from ...services.llm_provider import LLMProviderService + from ...core.database import SessionLocal + db = SessionLocal() + notifications_payload = {} + try: + llm_service = LLMProviderService(db) + providers = llm_service.get_all_providers() + llm_providers_list = [ + { + "id": p.id, + "provider_type": p.provider_type, + "name": p.name, + "base_url": p.base_url, + "api_key": "********", + "default_model": p.default_model, + "is_active": p.is_active + } for p in providers + ] + + config_record = db.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first() + if config_record and isinstance(config_record.payload, dict): + notifications_payload = config_record.payload.get("notifications", {}) or {} + finally: + db.close() + + normalized_llm = normalize_llm_settings(config.settings.llm) + + return ConsolidatedSettingsResponse( + environments=[env.dict() for env in config.environments], + connections=config.settings.connections, + llm=normalized_llm, + llm_providers=llm_providers_list, + logging=config.settings.logging.dict(), + storage=config.settings.storage.dict(), + notifications=notifications_payload + ) +# [/DEF:get_consolidated_settings:Function] + +# [DEF:update_consolidated_settings:Function] +# @TIER: STANDARD +# @PURPOSE: Bulk update application settings from the consolidated view. +# @PRE: User has admin permissions, config is valid. +# @POST: Settings are updated and saved via ConfigManager. +@router.patch("/consolidated") +async def update_consolidated_settings( + settings_patch: dict, + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("update_consolidated_settings"): + logger.info("[update_consolidated_settings][Entry] Applying consolidated settings patch") + + current_config = config_manager.get_config() + current_settings = current_config.settings + + # Update connections if provided + if "connections" in settings_patch: + current_settings.connections = settings_patch["connections"] + + # Update LLM if provided + if "llm" in settings_patch: + current_settings.llm = normalize_llm_settings(settings_patch["llm"]) + + # Update Logging if provided + if "logging" in settings_patch: + current_settings.logging = LoggingConfig(**settings_patch["logging"]) + + # Update Storage if provided + if "storage" in settings_patch: + new_storage = StorageConfig(**settings_patch["storage"]) + is_valid, message = config_manager.validate_path(new_storage.root_path) + if not is_valid: + raise HTTPException(status_code=400, detail=message) + current_settings.storage = new_storage + + if "notifications" in settings_patch: + payload = config_manager.get_payload() + payload["notifications"] = settings_patch["notifications"] + config_manager.save_config(payload) + + config_manager.update_global_settings(current_settings) + return {"status": "success", "message": "Settings updated"} +# [/DEF:update_consolidated_settings:Function] + +# [DEF:get_validation_policies:Function] +# @TIER: STANDARD +# @PURPOSE: Lists all validation policies. +# @RETURN: List[ValidationPolicyResponse] - List of policies. +@router.get("/automation/policies", response_model=List[ValidationPolicyResponse]) +async def get_validation_policies( + db: Session = Depends(get_db), + _ = Depends(has_permission("admin:settings", "READ")) +): + with belief_scope("get_validation_policies"): + return db.query(ValidationPolicy).all() +# [/DEF:get_validation_policies:Function] + +# [DEF:create_validation_policy:Function] +# @TIER: STANDARD +# @PURPOSE: Creates a new validation policy. +# @PARAM: policy (ValidationPolicyCreate) - The policy data. +# @RETURN: ValidationPolicyResponse - The created policy. +@router.post("/automation/policies", response_model=ValidationPolicyResponse) +async def create_validation_policy( + policy: ValidationPolicyCreate, + db: Session = Depends(get_db), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("create_validation_policy"): + db_policy = ValidationPolicy(**policy.dict()) + db.add(db_policy) + db.commit() + db.refresh(db_policy) + return db_policy +# [/DEF:create_validation_policy:Function] + +# [DEF:update_validation_policy:Function] +# @TIER: STANDARD +# @PURPOSE: Updates an existing validation policy. +# @PARAM: id (str) - The ID of the policy to update. +# @PARAM: policy (ValidationPolicyUpdate) - The updated policy data. +# @RETURN: ValidationPolicyResponse - The updated policy. +@router.patch("/automation/policies/{id}", response_model=ValidationPolicyResponse) +async def update_validation_policy( + id: str, + policy: ValidationPolicyUpdate, + db: Session = Depends(get_db), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("update_validation_policy"): + db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first() + if not db_policy: + raise HTTPException(status_code=404, detail="Policy not found") + + update_data = policy.dict(exclude_unset=True) + for key, value in update_data.items(): + setattr(db_policy, key, value) + + db.commit() + db.refresh(db_policy) + return db_policy +# [/DEF:update_validation_policy:Function] + +# [DEF:delete_validation_policy:Function] +# @TIER: STANDARD +# @PURPOSE: Deletes a validation policy. +# @PARAM: id (str) - The ID of the policy to delete. +@router.delete("/automation/policies/{id}") +async def delete_validation_policy( + id: str, + db: Session = Depends(get_db), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("delete_validation_policy"): + db_policy = db.query(ValidationPolicy).filter(ValidationPolicy.id == id).first() + if not db_policy: + raise HTTPException(status_code=404, detail="Policy not found") + + db.delete(db_policy) + db.commit() + return {"message": "Policy deleted"} +# [/DEF:delete_validation_policy:Function] + +# [/DEF:SettingsRouter:Module] diff --git a/backend/src/api/routes/storage.py b/backend/src/api/routes/storage.py index 98b0ef84..54bbde28 100644 --- a/backend/src/api/routes/storage.py +++ b/backend/src/api/routes/storage.py @@ -4,7 +4,7 @@ # @SEMANTICS: storage, files, upload, download, backup, repository # @PURPOSE: API endpoints for file storage management (backups and repositories). # @LAYER: API -# @RELATION: DEPENDS_ON -> backend.src.models.storage +# @RELATION: DEPENDS_ON -> [backend.src.models.storage] # # @INVARIANT: All paths must be validated against path traversal. @@ -22,6 +22,7 @@ from ...core.logger import belief_scope router = APIRouter(tags=["storage"]) # [DEF:list_files:Function] +# @TIER: STANDARD # @PURPOSE: List all files and directories in the storage system. # # @PRE: None. @@ -31,7 +32,7 @@ router = APIRouter(tags=["storage"]) # @PARAM: path (Optional[str]) - Subpath within the category. # @RETURN: List[StoredFile] - List of files/directories. # -# @RELATION: CALLS -> StoragePlugin.list_files +# @RELATION: CALLS -> [backend.src.plugins.storage.plugin.StoragePlugin.list_files] @router.get("/files", response_model=List[StoredFile]) async def list_files( category: Optional[FileCategory] = None, @@ -48,6 +49,7 @@ async def list_files( # [/DEF:list_files:Function] # [DEF:upload_file:Function] +# @TIER: STANDARD # @PURPOSE: Upload a file to the storage system. # # @PRE: category must be a valid FileCategory. @@ -61,7 +63,7 @@ async def list_files( # # @SIDE_EFFECT: Writes file to the filesystem. # -# @RELATION: CALLS -> StoragePlugin.save_file +# @RELATION: CALLS -> [backend.src.plugins.storage.plugin.StoragePlugin.save_file] @router.post("/upload", response_model=StoredFile, status_code=201) async def upload_file( category: FileCategory = Form(...), @@ -81,6 +83,7 @@ async def upload_file( # [/DEF:upload_file:Function] # [DEF:delete_file:Function] +# @TIER: STANDARD # @PURPOSE: Delete a specific file or directory. # # @PRE: category must be a valid FileCategory. @@ -92,7 +95,7 @@ async def upload_file( # # @SIDE_EFFECT: Deletes item from the filesystem. # -# @RELATION: CALLS -> StoragePlugin.delete_file +# @RELATION: CALLS -> [backend.src.plugins.storage.plugin.StoragePlugin.delete_file] @router.delete("/files/{category}/{path:path}", status_code=204) async def delete_file( category: FileCategory, @@ -113,6 +116,7 @@ async def delete_file( # [/DEF:delete_file:Function] # [DEF:download_file:Function] +# @TIER: STANDARD # @PURPOSE: Retrieve a file for download. # # @PRE: category must be a valid FileCategory. @@ -122,7 +126,7 @@ async def delete_file( # @PARAM: path (str) - Relative path of the file. # @RETURN: FileResponse - The file content. # -# @RELATION: CALLS -> StoragePlugin.get_file_path +# @RELATION: CALLS -> [backend.src.plugins.storage.plugin.StoragePlugin.get_file_path] @router.get("/download/{category}/{path:path}") async def download_file( category: FileCategory, @@ -145,6 +149,7 @@ async def download_file( # [/DEF:download_file:Function] # [DEF:get_file_by_path:Function] +# @TIER: STANDARD # @PURPOSE: Retrieve a file by validated absolute/relative path under storage root. # # @PRE: path must resolve under configured storage root. @@ -153,8 +158,8 @@ async def download_file( # @PARAM: path (str) - Absolute or storage-root-relative file path. # @RETURN: FileResponse - The file content. # -# @RELATION: CALLS -> StoragePlugin.get_storage_root -# @RELATION: CALLS -> StoragePlugin.validate_path +# @RELATION: CALLS -> [backend.src.plugins.storage.plugin.StoragePlugin.get_storage_root] +# @RELATION: CALLS -> [backend.src.plugins.storage.plugin.StoragePlugin.validate_path] @router.get("/file") async def get_file_by_path( path: str, diff --git a/backend/src/api/routes/tasks.py b/backend/src/api/routes/tasks.py index d6c648a0..deaf4933 100755 --- a/backend/src/api/routes/tasks.py +++ b/backend/src/api/routes/tasks.py @@ -1,348 +1,324 @@ -# [DEF:TasksRouter:Module] -# @TIER: STANDARD -# @SEMANTICS: api, router, tasks, create, list, get, logs -# @PURPOSE: Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks. -# @LAYER: UI (API) -# @RELATION: Depends on the TaskManager. It is included by the main app. -from typing import List, Dict, Any, Optional -from fastapi import APIRouter, Depends, HTTPException, status, Query -from pydantic import BaseModel -from ...core.logger import belief_scope - -from ...core.task_manager import TaskManager, Task, TaskStatus, LogEntry -from ...core.task_manager.models import LogFilter, LogStats -from ...dependencies import get_task_manager, has_permission, get_current_user, get_config_manager -from ...core.config_manager import ConfigManager -from ...services.llm_prompt_templates import ( - is_multimodal_model, - normalize_llm_settings, - resolve_bound_provider_id, -) - -router = APIRouter() - -TASK_TYPE_PLUGIN_MAP = { - "llm_validation": ["llm_dashboard_validation"], - "backup": ["superset-backup"], - "migration": ["superset-migration"], -} - -class CreateTaskRequest(BaseModel): - plugin_id: str - params: Dict[str, Any] - -class ResolveTaskRequest(BaseModel): - resolution_params: Dict[str, Any] - -class ResumeTaskRequest(BaseModel): - passwords: Dict[str, str] - -@router.post("", response_model=Task, status_code=status.HTTP_201_CREATED) -# [DEF:create_task:Function] -# @PURPOSE: Create and start a new task for a given plugin. -# @PARAM: request (CreateTaskRequest) - The request body containing plugin_id and params. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: plugin_id must exist and params must be valid for that plugin. -# @POST: A new task is created and started. -# @RETURN: Task - The created task instance. -async def create_task( - request: CreateTaskRequest, - task_manager: TaskManager = Depends(get_task_manager), - current_user = Depends(get_current_user), - config_manager: ConfigManager = Depends(get_config_manager), -): - # Dynamic permission check based on plugin_id - has_permission(f"plugin:{request.plugin_id}", "EXECUTE")(current_user) - """ - Create and start a new task for a given plugin. - """ - with belief_scope("create_task"): - try: - # Special handling for LLM tasks to resolve provider config by task binding. - if request.plugin_id in {"llm_dashboard_validation", "llm_documentation"}: - from ...core.database import SessionLocal - from ...services.llm_provider import LLMProviderService - db = SessionLocal() - try: - llm_service = LLMProviderService(db) - provider_id = request.params.get("provider_id") - if not provider_id: - llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm) - binding_key = "dashboard_validation" if request.plugin_id == "llm_dashboard_validation" else "documentation" - provider_id = resolve_bound_provider_id(llm_settings, binding_key) - if provider_id: - request.params["provider_id"] = provider_id - if not provider_id: - providers = llm_service.get_all_providers() - active_provider = next((p for p in providers if p.is_active), None) - if active_provider: - provider_id = active_provider.id - request.params["provider_id"] = provider_id - - if provider_id: - db_provider = llm_service.get_provider(provider_id) - if not db_provider: - raise ValueError(f"LLM Provider {provider_id} not found") - if request.plugin_id == "llm_dashboard_validation" and not is_multimodal_model( - db_provider.default_model, - db_provider.provider_type, - ): - raise HTTPException( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - detail="Selected provider model is not multimodal for dashboard validation", - ) - finally: - db.close() - - task = await task_manager.create_task( - plugin_id=request.plugin_id, - params=request.params - ) - return task - except ValueError as e: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) -# [/DEF:create_task:Function] - -@router.get("", response_model=List[Task]) -# [DEF:list_tasks:Function] -# @PURPOSE: Retrieve a list of tasks with pagination and optional status filter. -# @PARAM: limit (int) - Maximum number of tasks to return. -# @PARAM: offset (int) - Number of tasks to skip. -# @PARAM: status (Optional[TaskStatus]) - Filter by task status. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task_manager must be available. -# @POST: Returns a list of tasks. -# @RETURN: List[Task] - List of tasks. -async def list_tasks( - limit: int = 10, - offset: int = 0, - status_filter: Optional[TaskStatus] = Query(None, alias="status"), - task_type: Optional[str] = Query(None, description="Task category: llm_validation, backup, migration"), - plugin_id: Optional[List[str]] = Query(None, description="Filter by plugin_id (repeatable query param)"), - completed_only: bool = Query(False, description="Return only completed tasks (SUCCESS/FAILED)"), - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "READ")) -): - """ - Retrieve a list of tasks with pagination and optional status filter. - """ - with belief_scope("list_tasks"): - plugin_filters = list(plugin_id) if plugin_id else [] - if task_type: - if task_type not in TASK_TYPE_PLUGIN_MAP: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported task_type '{task_type}'. Allowed: {', '.join(TASK_TYPE_PLUGIN_MAP.keys())}" - ) - plugin_filters.extend(TASK_TYPE_PLUGIN_MAP[task_type]) - - return task_manager.get_tasks( - limit=limit, - offset=offset, - status=status_filter, - plugin_ids=plugin_filters or None, - completed_only=completed_only - ) -# [/DEF:list_tasks:Function] - -@router.get("/{task_id}", response_model=Task) -# [DEF:get_task:Function] -# @PURPOSE: Retrieve the details of a specific task. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task_id must exist. -# @POST: Returns task details or raises 404. -# @RETURN: Task - The task details. -async def get_task( - task_id: str, - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "READ")) -): - """ - Retrieve the details of a specific task. - """ - with belief_scope("get_task"): - task = task_manager.get_task(task_id) - if not task: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") - return task -# [/DEF:get_task:Function] - -@router.get("/{task_id}/logs", response_model=List[LogEntry]) -# [DEF:get_task_logs:Function] -# @PURPOSE: Retrieve logs for a specific task with optional filtering. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: level (Optional[str]) - Filter by log level (DEBUG, INFO, WARNING, ERROR). -# @PARAM: source (Optional[str]) - Filter by source component. -# @PARAM: search (Optional[str]) - Text search in message. -# @PARAM: offset (int) - Number of logs to skip. -# @PARAM: limit (int) - Maximum number of logs to return. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task_id must exist. -# @POST: Returns a list of log entries or raises 404. -# @RETURN: List[LogEntry] - List of log entries. -# @TIER: CRITICAL -# @TEST_CONTRACT get_task_logs_api -> -# { -# required_params: {task_id: str}, -# optional_params: {level: str, source: str, search: str}, -# invariants: ["returns 404 for non-existent task", "applies filters correctly"] -# } -# @TEST_FIXTURE valid_task_logs_request -> {"task_id": "test_1", "level": "INFO"} -# @TEST_EDGE task_not_found -> raises 404 -# @TEST_EDGE invalid_limit -> Query(limit=0) returns 422 -# @TEST_INVARIANT response_purity -> verifies: [valid_task_logs_request] -# @TEST_CONTRACT: TaskLogQueryInput -> List[LogEntry] -# @TEST_SCENARIO: existing_task_logs_filtered -> Returns filtered logs by level/source/search with pagination. -# @TEST_FIXTURE: valid_task_with_mixed_logs -> backend/tests/fixtures/task_logs/valid_task_with_mixed_logs.json -# @TEST_EDGE: missing_task -> Unknown task_id returns 404 Task not found. -# @TEST_EDGE: invalid_level_type -> Non-string/invalid level query rejected by validation or yields empty result. -# @TEST_EDGE: pagination_bounds -> offset=0 and limit=1000 remain within API bounds and do not overflow. -# @TEST_INVARIANT: logs_only_for_existing_task -> VERIFIED_BY: [existing_task_logs_filtered, missing_task] -async def get_task_logs( - task_id: str, - level: Optional[str] = Query(None, description="Filter by log level (DEBUG, INFO, WARNING, ERROR)"), - source: Optional[str] = Query(None, description="Filter by source component"), - search: Optional[str] = Query(None, description="Text search in message"), - offset: int = Query(0, ge=0, description="Number of logs to skip"), - limit: int = Query(100, ge=1, le=1000, description="Maximum number of logs to return"), - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "READ")) -): - """ - Retrieve logs for a specific task with optional filtering. - Supports filtering by level, source, and text search. - """ - with belief_scope("get_task_logs"): - task = task_manager.get_task(task_id) - if not task: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") - - log_filter = LogFilter( - level=level.upper() if level else None, - source=source, - search=search, - offset=offset, - limit=limit - ) - return task_manager.get_task_logs(task_id, log_filter) -# [/DEF:get_task_logs:Function] - -@router.get("/{task_id}/logs/stats", response_model=LogStats) -# [DEF:get_task_log_stats:Function] -# @PURPOSE: Get statistics about logs for a task (counts by level and source). -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task_id must exist. -# @POST: Returns log statistics or raises 404. -# @RETURN: LogStats - Statistics about task logs. -async def get_task_log_stats( - task_id: str, - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "READ")) -): - """ - Get statistics about logs for a task (counts by level and source). - """ - with belief_scope("get_task_log_stats"): - task = task_manager.get_task(task_id) - if not task: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") - return task_manager.get_task_log_stats(task_id) -# [/DEF:get_task_log_stats:Function] - -@router.get("/{task_id}/logs/sources", response_model=List[str]) -# [DEF:get_task_log_sources:Function] -# @PURPOSE: Get unique sources for a task's logs. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task_id must exist. -# @POST: Returns list of unique source names or raises 404. -# @RETURN: List[str] - Unique source names. -async def get_task_log_sources( - task_id: str, - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "READ")) -): - """ - Get unique sources for a task's logs. - """ - with belief_scope("get_task_log_sources"): - task = task_manager.get_task(task_id) - if not task: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") - return task_manager.get_task_log_sources(task_id) -# [/DEF:get_task_log_sources:Function] - -@router.post("/{task_id}/resolve", response_model=Task) -# [DEF:resolve_task:Function] -# @PURPOSE: Resolve a task that is awaiting mapping. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: request (ResolveTaskRequest) - The resolution parameters. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task must be in AWAITING_MAPPING status. -# @POST: Task is resolved and resumes execution. -# @RETURN: Task - The updated task object. -async def resolve_task( - task_id: str, - request: ResolveTaskRequest, - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "WRITE")) -): - """ - Resolve a task that is awaiting mapping. - """ - with belief_scope("resolve_task"): - try: - await task_manager.resolve_task(task_id, request.resolution_params) - return task_manager.get_task(task_id) - except ValueError as e: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:resolve_task:Function] - -@router.post("/{task_id}/resume", response_model=Task) -# [DEF:resume_task:Function] -# @PURPOSE: Resume a task that is awaiting input (e.g., passwords). -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: request (ResumeTaskRequest) - The input (passwords). -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task must be in AWAITING_INPUT status. -# @POST: Task resumes execution with provided input. -# @RETURN: Task - The updated task object. -async def resume_task( - task_id: str, - request: ResumeTaskRequest, - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "WRITE")) -): - """ - Resume a task that is awaiting input (e.g., passwords). - """ - with belief_scope("resume_task"): - try: - task_manager.resume_task_with_password(task_id, request.passwords) - return task_manager.get_task(task_id) - except ValueError as e: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) -# [/DEF:resume_task:Function] - -@router.delete("", status_code=status.HTTP_204_NO_CONTENT) -# [DEF:clear_tasks:Function] -# @PURPOSE: Clear tasks matching the status filter. -# @PARAM: status (Optional[TaskStatus]) - Filter by task status. -# @PARAM: task_manager (TaskManager) - The task manager instance. -# @PRE: task_manager is available. -# @POST: Tasks are removed from memory/persistence. -async def clear_tasks( - status: Optional[TaskStatus] = None, - task_manager: TaskManager = Depends(get_task_manager), - _ = Depends(has_permission("tasks", "WRITE")) -): - """ - Clear tasks matching the status filter. If no filter, clears all non-running tasks. - """ - with belief_scope("clear_tasks", f"status={status}"): - task_manager.clear_tasks(status) - return -# [/DEF:clear_tasks:Function] -# [/DEF:TasksRouter:Module] +# [DEF:TasksRouter:Module] +# @TIER: STANDARD +# @SEMANTICS: api, router, tasks, create, list, get, logs +# @PURPOSE: Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks. +# @LAYER: UI (API) +# @RELATION: DEPENDS_ON -> [backend.src.core.task_manager.manager.TaskManager] +# @RELATION: DEPENDS_ON -> [backend.src.core.config_manager.ConfigManager] +# @RELATION: DEPENDS_ON -> [backend.src.services.llm_provider.LLMProviderService] + +# [SECTION: IMPORTS] +from typing import List, Dict, Any, Optional +from fastapi import APIRouter, Depends, HTTPException, status, Query +from pydantic import BaseModel +from ...core.logger import belief_scope + +from ...core.task_manager import TaskManager, Task, TaskStatus, LogEntry +from ...core.task_manager.models import LogFilter, LogStats +from ...dependencies import get_task_manager, has_permission, get_current_user, get_config_manager +from ...core.config_manager import ConfigManager +from ...services.llm_prompt_templates import ( + is_multimodal_model, + normalize_llm_settings, + resolve_bound_provider_id, +) +# [/SECTION] + +router = APIRouter() + +TASK_TYPE_PLUGIN_MAP = { + "llm_validation": ["llm_dashboard_validation"], + "backup": ["superset-backup"], + "migration": ["superset-migration"], +} + +class CreateTaskRequest(BaseModel): + plugin_id: str + params: Dict[str, Any] + +class ResolveTaskRequest(BaseModel): + resolution_params: Dict[str, Any] + +class ResumeTaskRequest(BaseModel): + passwords: Dict[str, str] + +# [DEF:create_task:Function] +# @TIER: STANDARD +# @PURPOSE: Create and start a new task for a given plugin. +# @PARAM: request (CreateTaskRequest) - The request body containing plugin_id and params. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: plugin_id must exist and params must be valid for that plugin. +# @POST: A new task is created and started. +# @RETURN: Task - The created task instance. +@router.post("", response_model=Task, status_code=status.HTTP_201_CREATED) +async def create_task( + request: CreateTaskRequest, + task_manager: TaskManager = Depends(get_task_manager), + current_user = Depends(get_current_user), + config_manager: ConfigManager = Depends(get_config_manager), +): + # Dynamic permission check based on plugin_id + has_permission(f"plugin:{request.plugin_id}", "EXECUTE")(current_user) + with belief_scope("create_task"): + try: + # Special handling for LLM tasks to resolve provider config by task binding. + if request.plugin_id in {"llm_dashboard_validation", "llm_documentation"}: + from ...core.database import SessionLocal + from ...services.llm_provider import LLMProviderService + db = SessionLocal() + try: + llm_service = LLMProviderService(db) + provider_id = request.params.get("provider_id") + if not provider_id: + llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm) + binding_key = "dashboard_validation" if request.plugin_id == "llm_dashboard_validation" else "documentation" + provider_id = resolve_bound_provider_id(llm_settings, binding_key) + if provider_id: + request.params["provider_id"] = provider_id + if not provider_id: + providers = llm_service.get_all_providers() + active_provider = next((p for p in providers if p.is_active), None) + if active_provider: + provider_id = active_provider.id + request.params["provider_id"] = provider_id + + if provider_id: + db_provider = llm_service.get_provider(provider_id) + if not db_provider: + raise ValueError(f"LLM Provider {provider_id} not found") + if request.plugin_id == "llm_dashboard_validation" and not is_multimodal_model( + db_provider.default_model, + db_provider.provider_type, + ): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Selected provider model is not multimodal for dashboard validation", + ) + finally: + db.close() + + task = await task_manager.create_task( + plugin_id=request.plugin_id, + params=request.params + ) + return task + except ValueError as e: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) +# [/DEF:create_task:Function] + +# [DEF:list_tasks:Function] +# @TIER: STANDARD +# @PURPOSE: Retrieve a list of tasks with pagination and optional status filter. +# @PARAM: limit (int) - Maximum number of tasks to return. +# @PARAM: offset (int) - Number of tasks to skip. +# @PARAM: status (Optional[TaskStatus]) - Filter by task status. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task_manager must be available. +# @POST: Returns a list of tasks. +# @RETURN: List[Task] - List of tasks. +@router.get("", response_model=List[Task]) +async def list_tasks( + limit: int = 10, + offset: int = 0, + status_filter: Optional[TaskStatus] = Query(None, alias="status"), + task_type: Optional[str] = Query(None, description="Task category: llm_validation, backup, migration"), + plugin_id: Optional[List[str]] = Query(None, description="Filter by plugin_id (repeatable query param)"), + completed_only: bool = Query(False, description="Return only completed tasks (SUCCESS/FAILED)"), + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "READ")) +): + with belief_scope("list_tasks"): + plugin_filters = list(plugin_id) if plugin_id else [] + if task_type: + if task_type not in TASK_TYPE_PLUGIN_MAP: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported task_type '{task_type}'. Allowed: {', '.join(TASK_TYPE_PLUGIN_MAP.keys())}" + ) + plugin_filters.extend(TASK_TYPE_PLUGIN_MAP[task_type]) + + return task_manager.get_tasks( + limit=limit, + offset=offset, + status=status_filter, + plugin_ids=plugin_filters or None, + completed_only=completed_only + ) +# [/DEF:list_tasks:Function] + +# [DEF:get_task:Function] +# @TIER: STANDARD +# @PURPOSE: Retrieve the details of a specific task. +# @PARAM: task_id (str) - The unique identifier of the task. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task_id must exist. +# @POST: Returns task details or raises 404. +# @RETURN: Task - The task details. +@router.get("/{task_id}", response_model=Task) +async def get_task( + task_id: str, + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "READ")) +): + with belief_scope("get_task"): + task = task_manager.get_task(task_id) + if not task: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") + return task +# [/DEF:get_task:Function] + +# [DEF:get_task_logs:Function] +# @TIER: CRITICAL +# @PURPOSE: Retrieve logs for a specific task with optional filtering. +# @PARAM: task_id (str) - The unique identifier of the task. +# @PARAM: level (Optional[str]) - Filter by log level (DEBUG, INFO, WARNING, ERROR). +# @PARAM: source (Optional[str]) - Filter by source component. +# @PARAM: search (Optional[str]) - Text search in message. +# @PARAM: offset (int) - Number of logs to skip. +# @PARAM: limit (int) - Maximum number of logs to return. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task_id must exist. +# @POST: Returns a list of log entries or raises 404. +# @RETURN: List[LogEntry] - List of log entries. +# @TEST_CONTRACT: TaskLogQueryInput -> List[LogEntry] +# @TEST_SCENARIO: existing_task_logs_filtered -> Returns filtered logs by level/source/search with pagination. +# @TEST_FIXTURE: valid_task_with_mixed_logs -> backend/tests/fixtures/task_logs/valid_task_with_mixed_logs.json +# @TEST_EDGE: missing_task -> Unknown task_id returns 404 Task not found. +# @TEST_EDGE: invalid_level_type -> Non-string/invalid level query rejected by validation or yields empty result. +# @TEST_EDGE: pagination_bounds -> offset=0 and limit=1000 remain within API bounds and do not overflow. +# @TEST_INVARIANT: logs_only_for_existing_task -> VERIFIED_BY: [existing_task_logs_filtered, missing_task] +@router.get("/{task_id}/logs", response_model=List[LogEntry]) +async def get_task_logs( + task_id: str, + level: Optional[str] = Query(None, description="Filter by log level (DEBUG, INFO, WARNING, ERROR)"), + source: Optional[str] = Query(None, description="Filter by source component"), + search: Optional[str] = Query(None, description="Text search in message"), + offset: int = Query(0, ge=0, description="Number of logs to skip"), + limit: int = Query(100, ge=1, le=1000, description="Maximum number of logs to return"), + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "READ")) +): + with belief_scope("get_task_logs"): + task = task_manager.get_task(task_id) + if not task: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") + + log_filter = LogFilter( + level=level.upper() if level else None, + source=source, + search=search, + offset=offset, + limit=limit + ) + return task_manager.get_task_logs(task_id, log_filter) +# [/DEF:get_task_logs:Function] + +# [DEF:get_task_log_stats:Function] +# @TIER: STANDARD +# @PURPOSE: Get statistics about logs for a task (counts by level and source). +# @PARAM: task_id (str) - The unique identifier of the task. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task_id must exist. +# @POST: Returns log statistics or raises 404. +# @RETURN: LogStats - Statistics about task logs. +@router.get("/{task_id}/logs/stats", response_model=LogStats) +async def get_task_log_stats( + task_id: str, + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "READ")) +): + with belief_scope("get_task_log_stats"): + task = task_manager.get_task(task_id) + if not task: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") + return task_manager.get_task_log_stats(task_id) +# [/DEF:get_task_log_stats:Function] + +# [DEF:get_task_log_sources:Function] +# @TIER: STANDARD +# @PURPOSE: Get unique sources for a task's logs. +# @PARAM: task_id (str) - The unique identifier of the task. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task_id must exist. +# @POST: Returns list of unique source names or raises 404. +# @RETURN: List[str] - Unique source names. +@router.get("/{task_id}/logs/sources", response_model=List[str]) +async def get_task_log_sources( + task_id: str, + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "READ")) +): + with belief_scope("get_task_log_sources"): + task = task_manager.get_task(task_id) + if not task: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found") + return task_manager.get_task_log_sources(task_id) +# [/DEF:get_task_log_sources:Function] + +# [DEF:resolve_task:Function] +# @TIER: STANDARD +# @PURPOSE: Resolve a task that is awaiting mapping. +# @PARAM: task_id (str) - The unique identifier of the task. +# @PARAM: request (ResolveTaskRequest) - The resolution parameters. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task must be in AWAITING_MAPPING status. +# @POST: Task is resolved and resumes execution. +# @RETURN: Task - The updated task object. +@router.post("/{task_id}/resolve", response_model=Task) +async def resolve_task( + task_id: str, + request: ResolveTaskRequest, + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "WRITE")) +): + with belief_scope("resolve_task"): + try: + await task_manager.resolve_task(task_id, request.resolution_params) + return task_manager.get_task(task_id) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:resolve_task:Function] + +# [DEF:resume_task:Function] +# @TIER: STANDARD +# @PURPOSE: Resume a task that is awaiting input (e.g., passwords). +# @PARAM: task_id (str) - The unique identifier of the task. +# @PARAM: request (ResumeTaskRequest) - The input (passwords). +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task must be in AWAITING_INPUT status. +# @POST: Task resumes execution with provided input. +# @RETURN: Task - The updated task object. +@router.post("/{task_id}/resume", response_model=Task) +async def resume_task( + task_id: str, + request: ResumeTaskRequest, + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "WRITE")) +): + with belief_scope("resume_task"): + try: + task_manager.resume_task_with_password(task_id, request.passwords) + return task_manager.get_task(task_id) + except ValueError as e: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) +# [/DEF:resume_task:Function] + +# [DEF:clear_tasks:Function] +# @TIER: STANDARD +# @PURPOSE: Clear tasks matching the status filter. +# @PARAM: status (Optional[TaskStatus]) - Filter by task status. +# @PARAM: task_manager (TaskManager) - The task manager instance. +# @PRE: task_manager is available. +# @POST: Tasks are removed from memory/persistence. +@router.delete("", status_code=status.HTTP_204_NO_CONTENT) +async def clear_tasks( + status: Optional[TaskStatus] = None, + task_manager: TaskManager = Depends(get_task_manager), + _ = Depends(has_permission("tasks", "WRITE")) +): + with belief_scope("clear_tasks", f"status={status}"): + task_manager.clear_tasks(status) + return +# [/DEF:clear_tasks:Function] + +# [/DEF:TasksRouter:Module] diff --git a/backend/src/app.py b/backend/src/app.py index 5e5eab9f..b4587448 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -1,21 +1,27 @@ -# [DEF:AppModule:Module] -# @TIER: CRITICAL -# @SEMANTICS: app, main, entrypoint, fastapi -# @PURPOSE: The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming. -# @LAYER: UI (API) -# @RELATION: Depends on the dependency module and API route modules. -# @INVARIANT: Only one FastAPI app instance exists per process. -# @INVARIANT: All WebSocket connections must be properly cleaned up on disconnect. -from pathlib import Path - -# project_root is used for static files mounting -project_root = Path(__file__).resolve().parent.parent.parent - -from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPException -from starlette.middleware.sessions import SessionMiddleware -from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse +# [DEF:AppModule:Module] +# @TIER: CRITICAL +# @SEMANTICS: app, main, entrypoint, fastapi +# @PURPOSE: The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming. +# @LAYER: UI (API) +# @RELATION: DEPENDS_ON ->[backend.src.dependencies] +# @RELATION: DEPENDS_ON ->[backend.src.api.routes] +# @INVARIANT: Only one FastAPI app instance exists per process. +# @INVARIANT: All WebSocket connections must be properly cleaned up on disconnect. +# @PRE: Python environment and dependencies installed; configuration database available. +# @POST: FastAPI app instance is created, middleware configured, and routes registered. +# @SIDE_EFFECT: Starts background scheduler and binds network ports for HTTP/WS traffic. +# @DATA_CONTRACT: [HTTP Request | WS Message] -> [HTTP Response | JSON Log Stream] + +from pathlib import Path + +# project_root is used for static files mounting +project_root = Path(__file__).resolve().parent.parent.parent + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect, Request, HTTPException +from starlette.middleware.sessions import SessionMiddleware +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from fastapi.responses import FileResponse import asyncio from .dependencies import get_task_manager, get_scheduler_service @@ -24,22 +30,24 @@ from .core.utils.network import NetworkError from .core.logger import logger, belief_scope from .api.routes import plugins, tasks, settings, environments, mappings, migration, connections, git, storage, admin, llm, dashboards, datasets, reports, assistant, clean_release, clean_release_v2, profile, health from .api import auth - -# [DEF:App:Global] -# @SEMANTICS: app, fastapi, instance -# @PURPOSE: The global FastAPI application instance. -app = FastAPI( - title="Superset Tools API", - description="API for managing Superset automation tools and plugins.", - version="1.0.0", -) -# [/DEF:App:Global] - -# [DEF:startup_event:Function] -# @PURPOSE: Handles application startup tasks, such as starting the scheduler. -# @PRE: None. -# @POST: Scheduler is started. -# Startup event + +# [DEF:App:Global] +# @TIER: TRIVIAL +# @SEMANTICS: app, fastapi, instance +# @PURPOSE: The global FastAPI application instance. +app = FastAPI( + title="Superset Tools API", + description="API for managing Superset automation tools and plugins.", + version="1.0.0", +) +# [/DEF:App:Global] + +# [DEF:startup_event:Function] +# @TIER: STANDARD +# @PURPOSE: Handles application startup tasks, such as starting the scheduler. +# @PRE: None. +# @POST: Scheduler is started. +# Startup event @app.on_event("startup") async def startup_event(): with belief_scope("startup_event"): @@ -47,259 +55,268 @@ async def startup_event(): scheduler = get_scheduler_service() scheduler.start() # [/DEF:startup_event:Function] - -# [DEF:shutdown_event:Function] -# @PURPOSE: Handles application shutdown tasks, such as stopping the scheduler. -# @PRE: None. -# @POST: Scheduler is stopped. -# Shutdown event -@app.on_event("shutdown") -async def shutdown_event(): - with belief_scope("shutdown_event"): - scheduler = get_scheduler_service() - scheduler.stop() -# [/DEF:shutdown_event:Function] - -# Configure Session Middleware (required by Authlib for OAuth2 flow) -from .core.auth.config import auth_config -app.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY) - -# Configure CORS -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], # Adjust this in production - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - - -# [DEF:network_error_handler:Function] -# @PURPOSE: Global exception handler for NetworkError. -# @PRE: request is a FastAPI Request object. -# @POST: Returns 503 HTTP Exception. -# @PARAM: request (Request) - The incoming request object. -# @PARAM: exc (NetworkError) - The exception instance. -@app.exception_handler(NetworkError) -async def network_error_handler(request: Request, exc: NetworkError): - with belief_scope("network_error_handler"): - logger.error(f"Network error: {exc}") - return HTTPException( - status_code=503, - detail="Environment unavailable. Please check if the Superset instance is running." - ) -# [/DEF:network_error_handler:Function] - -# [DEF:log_requests:Function] -# @PURPOSE: Middleware to log incoming HTTP requests and their response status. -# @PRE: request is a FastAPI Request object. -# @POST: Logs request and response details. -# @PARAM: request (Request) - The incoming request object. -# @PARAM: call_next (Callable) - The next middleware or route handler. -@app.middleware("http") -async def log_requests(request: Request, call_next): - with belief_scope("log_requests"): - # Avoid spamming logs for polling endpoints - is_polling = request.url.path.endswith("/api/tasks") and request.method == "GET" - - if not is_polling: - logger.info(f"Incoming request: {request.method} {request.url.path}") - - try: - response = await call_next(request) - if not is_polling: - logger.info(f"Response status: {response.status_code} for {request.url.path}") - return response - except NetworkError as e: - logger.error(f"Network error caught in middleware: {e}") - raise HTTPException( - status_code=503, - detail="Environment unavailable. Please check if the Superset instance is running." - ) -# [/DEF:log_requests:Function] - -# Include API routes -app.include_router(auth.router) -app.include_router(admin.router) -app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"]) -app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"]) -app.include_router(settings.router, prefix="/api/settings", tags=["Settings"]) -app.include_router(connections.router, prefix="/api/settings/connections", tags=["Connections"]) -app.include_router(environments.router, tags=["Environments"]) -app.include_router(mappings.router, prefix="/api/mappings", tags=["Mappings"]) -app.include_router(migration.router) -app.include_router(git.router, prefix="/api/git", tags=["Git"]) -app.include_router(llm.router, prefix="/api/llm", tags=["LLM"]) -app.include_router(storage.router, prefix="/api/storage", tags=["Storage"]) -app.include_router(dashboards.router) -app.include_router(datasets.router) -app.include_router(reports.router) -app.include_router(assistant.router, prefix="/api/assistant", tags=["Assistant"]) -app.include_router(clean_release.router) -app.include_router(clean_release_v2.router) -app.include_router(profile.router) -app.include_router(health.router) - - -# [DEF:api.include_routers:Action] -# @PURPOSE: Registers all API routers with the FastAPI application. -# @LAYER: API -# @SEMANTICS: routes, registration, api -# [/DEF:api.include_routers:Action] - -# [DEF:websocket_endpoint:Function] -# @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering. -# @PRE: task_id must be a valid task ID. -# @POST: WebSocket connection is managed and logs are streamed until disconnect. -# @TIER: CRITICAL -# @UX_STATE: Connecting -> Streaming -> (Disconnected) -# -# @TEST_CONTRACT: WebSocketLogStreamApi -> -# { -# required_fields: {websocket: WebSocket, task_id: str}, -# optional_fields: {source: str, level: str}, -# invariants: [ -# "Accepts the WebSocket connection", -# "Applies source and level filters correctly to streamed logs", -# "Cleans up subscriptions on disconnect" -# ] -# } -# @TEST_FIXTURE: valid_ws_connection -> {"task_id": "test_1", "source": "plugin"} -# @TEST_EDGE: task_not_found_ws -> closes connection or sends error -# @TEST_EDGE: empty_task_logs -> waits for new logs -# @TEST_INVARIANT: consistent_streaming -> verifies: [valid_ws_connection] -@app.websocket("/ws/logs/{task_id}") -async def websocket_endpoint( - websocket: WebSocket, - task_id: str, - source: str = None, - level: str = None -): - """ - WebSocket endpoint for real-time log streaming with optional server-side filtering. - - Query Parameters: - source: Filter logs by source component (e.g., "plugin", "superset_api") - level: Filter logs by minimum level (DEBUG, INFO, WARNING, ERROR) - """ - with belief_scope("websocket_endpoint", f"task_id={task_id}"): - await websocket.accept() - - # Normalize filter parameters - source_filter = source.lower() if source else None - level_filter = level.upper() if level else None - - # Level hierarchy for filtering - level_hierarchy = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3} - min_level = level_hierarchy.get(level_filter, 0) if level_filter else 0 - - logger.info(f"WebSocket connection accepted for task {task_id} (source={source_filter}, level={level_filter})") - task_manager = get_task_manager() - queue = await task_manager.subscribe_logs(task_id) - - def matches_filters(log_entry) -> bool: - """Check if log entry matches the filter criteria.""" - # Check source filter - if source_filter and log_entry.source.lower() != source_filter: - return False - - # Check level filter - if level_filter: - log_level = level_hierarchy.get(log_entry.level.upper(), 0) - if log_level < min_level: - return False - - return True - - try: - # Stream new logs - logger.info(f"Starting log stream for task {task_id}") - - # Send initial logs first to build context (apply filters) - initial_logs = task_manager.get_task_logs(task_id) - for log_entry in initial_logs: - if matches_filters(log_entry): - log_dict = log_entry.dict() - log_dict['timestamp'] = log_dict['timestamp'].isoformat() - await websocket.send_json(log_dict) - - # Force a check for AWAITING_INPUT status immediately upon connection - # This ensures that if the task is already waiting when the user connects, they get the prompt. - task = task_manager.get_task(task_id) - if task and task.status == "AWAITING_INPUT" and task.input_request: - # Construct a synthetic log entry to trigger the frontend handler - # This is a bit of a hack but avoids changing the websocket protocol significantly - synthetic_log = { - "timestamp": task.logs[-1].timestamp.isoformat() if task.logs else "2024-01-01T00:00:00", - "level": "INFO", - "message": "Task paused for user input (Connection Re-established)", - "context": {"input_request": task.input_request} - } - await websocket.send_json(synthetic_log) - - while True: - log_entry = await queue.get() - - # Apply server-side filtering - if not matches_filters(log_entry): - continue - - log_dict = log_entry.dict() - log_dict['timestamp'] = log_dict['timestamp'].isoformat() - await websocket.send_json(log_dict) - - # If task is finished, we could potentially close the connection - # but let's keep it open for a bit or until the client disconnects - if "Task completed successfully" in log_entry.message or "Task failed" in log_entry.message: - # Wait a bit to ensure client receives the last message - await asyncio.sleep(2) - # DO NOT BREAK here - allow client to keep connection open if they want to review logs - # or until they disconnect. Breaking closes the socket immediately. - # break - - except WebSocketDisconnect: - logger.info(f"WebSocket connection disconnected for task {task_id}") - except Exception as e: - logger.error(f"WebSocket error for task {task_id}: {e}") - finally: - task_manager.unsubscribe_logs(task_id, queue) -# [/DEF:websocket_endpoint:Function] - -# [DEF:StaticFiles:Mount] -# @SEMANTICS: static, frontend, spa -# @PURPOSE: Mounts the frontend build directory to serve static assets. -frontend_path = project_root / "frontend" / "build" -if frontend_path.exists(): - app.mount("/_app", StaticFiles(directory=str(frontend_path / "_app")), name="static") - - # [DEF:serve_spa:Function] - # @PURPOSE: Serves the SPA frontend for any path not matched by API routes. - # @PRE: frontend_path exists. - # @POST: Returns the requested file or index.html. - @app.get("/{file_path:path}", include_in_schema=False) - async def serve_spa(file_path: str): - with belief_scope("serve_spa"): - # Only serve SPA for non-API paths - # API routes are registered separately and should be matched by FastAPI first - if file_path and (file_path.startswith("api/") or file_path.startswith("/api/") or file_path == "api"): - # This should not happen if API routers are properly registered - # Return 404 instead of serving HTML - raise HTTPException(status_code=404, detail=f"API endpoint not found: {file_path}") - - full_path = frontend_path / file_path - if file_path and full_path.is_file(): - return FileResponse(str(full_path)) - return FileResponse(str(frontend_path / "index.html")) - # [/DEF:serve_spa:Function] -else: - # [DEF:read_root:Function] - # @PURPOSE: A simple root endpoint to confirm that the API is running when frontend is missing. - # @PRE: None. - # @POST: Returns a JSON message indicating API status. - @app.get("/") - async def read_root(): - with belief_scope("read_root"): - return {"message": "Superset Tools API is running (Frontend build not found)"} - # [/DEF:read_root:Function] -# [/DEF:StaticFiles:Mount] -# [/DEF:AppModule:Module] + +# [DEF:shutdown_event:Function] +# @TIER: STANDARD +# @PURPOSE: Handles application shutdown tasks, such as stopping the scheduler. +# @PRE: None. +# @POST: Scheduler is stopped. +# Shutdown event +@app.on_event("shutdown") +async def shutdown_event(): + with belief_scope("shutdown_event"): + scheduler = get_scheduler_service() + scheduler.stop() +# [/DEF:shutdown_event:Function] + +# Configure Session Middleware (required by Authlib for OAuth2 flow) +from .core.auth.config import auth_config +app.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY) + +# Configure CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # Adjust this in production + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +# [DEF:network_error_handler:Function] +# @TIER: TRIVIAL +# @PURPOSE: Global exception handler for NetworkError. +# @PRE: request is a FastAPI Request object. +# @POST: Returns 503 HTTP Exception. +# @PARAM: request (Request) - The incoming request object. +# @PARAM: exc (NetworkError) - The exception instance. +@app.exception_handler(NetworkError) +async def network_error_handler(request: Request, exc: NetworkError): + with belief_scope("network_error_handler"): + logger.error(f"Network error: {exc}") + return HTTPException( + status_code=503, + detail="Environment unavailable. Please check if the Superset instance is running." + ) +# [/DEF:network_error_handler:Function] + +# [DEF:log_requests:Function] +# @TIER: STANDARD +# @PURPOSE: Middleware to log incoming HTTP requests and their response status. +# @PRE: request is a FastAPI Request object. +# @POST: Logs request and response details. +# @PARAM: request (Request) - The incoming request object. +# @PARAM: call_next (Callable) - The next middleware or route handler. +@app.middleware("http") +async def log_requests(request: Request, call_next): + with belief_scope("log_requests"): + # Avoid spamming logs for polling endpoints + is_polling = request.url.path.endswith("/api/tasks") and request.method == "GET" + + if not is_polling: + logger.info(f"Incoming request: {request.method} {request.url.path}") + + try: + response = await call_next(request) + if not is_polling: + logger.info(f"Response status: {response.status_code} for {request.url.path}") + return response + except NetworkError as e: + logger.error(f"Network error caught in middleware: {e}") + raise HTTPException( + status_code=503, + detail="Environment unavailable. Please check if the Superset instance is running." + ) +# [/DEF:log_requests:Function] + +# Include API routes +app.include_router(auth.router) +app.include_router(admin.router) +app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"]) +app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"]) +app.include_router(settings.router, prefix="/api/settings", tags=["Settings"]) +app.include_router(connections.router, prefix="/api/settings/connections", tags=["Connections"]) +app.include_router(environments.router, tags=["Environments"]) +app.include_router(mappings.router, prefix="/api/mappings", tags=["Mappings"]) +app.include_router(migration.router) +app.include_router(git.router, prefix="/api/git", tags=["Git"]) +app.include_router(llm.router, prefix="/api/llm", tags=["LLM"]) +app.include_router(storage.router, prefix="/api/storage", tags=["Storage"]) +app.include_router(dashboards.router) +app.include_router(datasets.router) +app.include_router(reports.router) +app.include_router(assistant.router, prefix="/api/assistant", tags=["Assistant"]) +app.include_router(clean_release.router) +app.include_router(clean_release_v2.router) +app.include_router(profile.router) +app.include_router(health.router) + + +# [DEF:api.include_routers:Action] +# @TIER: TRIVIAL +# @PURPOSE: Registers all API routers with the FastAPI application. +# @LAYER: API +# @SEMANTICS: routes, registration, api +# [/DEF:api.include_routers:Action] + +# [DEF:websocket_endpoint:Function] +# @TIER: CRITICAL +# @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering. +# @PRE: task_id must be a valid task ID. +# @POST: WebSocket connection is managed and logs are streamed until disconnect. +# @SIDE_EFFECT: Subscribes to TaskManager log queue and broadcasts messages over network. +# @DATA_CONTRACT: [task_id: str, source: str, level: str] -> [JSON log entry objects] +# @UX_STATE: Connecting -> Streaming -> (Disconnected) +# +# @TEST_CONTRACT: WebSocketLogStreamApi -> +# { +# required_fields: {websocket: WebSocket, task_id: str}, +# optional_fields: {source: str, level: str}, +# invariants: [ +# "Accepts the WebSocket connection", +# "Applies source and level filters correctly to streamed logs", +# "Cleans up subscriptions on disconnect" +# ] +# } +# @TEST_FIXTURE: valid_ws_connection -> {"task_id": "test_1", "source": "plugin"} +# @TEST_EDGE: task_not_found_ws -> closes connection or sends error +# @TEST_EDGE: empty_task_logs -> waits for new logs +# @TEST_INVARIANT: consistent_streaming -> verifies: [valid_ws_connection] +@app.websocket("/ws/logs/{task_id}") +async def websocket_endpoint( + websocket: WebSocket, + task_id: str, + source: str = None, + level: str = None +): + """ + WebSocket endpoint for real-time log streaming with optional server-side filtering. + + Query Parameters: + source: Filter logs by source component (e.g., "plugin", "superset_api") + level: Filter logs by minimum level (DEBUG, INFO, WARNING, ERROR) + """ + with belief_scope("websocket_endpoint", f"task_id={task_id}"): + await websocket.accept() + + # Normalize filter parameters + source_filter = source.lower() if source else None + level_filter = level.upper() if level else None + + # Level hierarchy for filtering + level_hierarchy = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3} + min_level = level_hierarchy.get(level_filter, 0) if level_filter else 0 + + logger.info(f"WebSocket connection accepted for task {task_id} (source={source_filter}, level={level_filter})") + task_manager = get_task_manager() + queue = await task_manager.subscribe_logs(task_id) + + def matches_filters(log_entry) -> bool: + """Check if log entry matches the filter criteria.""" + # Check source filter + if source_filter and log_entry.source.lower() != source_filter: + return False + + # Check level filter + if level_filter: + log_level = level_hierarchy.get(log_entry.level.upper(), 0) + if log_level < min_level: + return False + + return True + + try: + # Stream new logs + logger.info(f"Starting log stream for task {task_id}") + + # Send initial logs first to build context (apply filters) + initial_logs = task_manager.get_task_logs(task_id) + for log_entry in initial_logs: + if matches_filters(log_entry): + log_dict = log_entry.dict() + log_dict['timestamp'] = log_dict['timestamp'].isoformat() + await websocket.send_json(log_dict) + + # Force a check for AWAITING_INPUT status immediately upon connection + # This ensures that if the task is already waiting when the user connects, they get the prompt. + task = task_manager.get_task(task_id) + if task and task.status == "AWAITING_INPUT" and task.input_request: + # Construct a synthetic log entry to trigger the frontend handler + # This is a bit of a hack but avoids changing the websocket protocol significantly + synthetic_log = { + "timestamp": task.logs[-1].timestamp.isoformat() if task.logs else "2024-01-01T00:00:00", + "level": "INFO", + "message": "Task paused for user input (Connection Re-established)", + "context": {"input_request": task.input_request} + } + await websocket.send_json(synthetic_log) + + while True: + log_entry = await queue.get() + + # Apply server-side filtering + if not matches_filters(log_entry): + continue + + log_dict = log_entry.dict() + log_dict['timestamp'] = log_dict['timestamp'].isoformat() + await websocket.send_json(log_dict) + + # If task is finished, we could potentially close the connection + # but let's keep it open for a bit or until the client disconnects + if "Task completed successfully" in log_entry.message or "Task failed" in log_entry.message: + # Wait a bit to ensure client receives the last message + await asyncio.sleep(2) + # DO NOT BREAK here - allow client to keep connection open if they want to review logs + # or until they disconnect. Breaking closes the socket immediately. + # break + + except WebSocketDisconnect: + logger.info(f"WebSocket connection disconnected for task {task_id}") + except Exception as e: + logger.error(f"WebSocket error for task {task_id}: {e}") + finally: + task_manager.unsubscribe_logs(task_id, queue) +# [/DEF:websocket_endpoint:Function] + +# [DEF:StaticFiles:Mount] +# @TIER: TRIVIAL +# @SEMANTICS: static, frontend, spa +# @PURPOSE: Mounts the frontend build directory to serve static assets. +frontend_path = project_root / "frontend" / "build" +if frontend_path.exists(): + app.mount("/_app", StaticFiles(directory=str(frontend_path / "_app")), name="static") + + # [DEF:serve_spa:Function] + # @TIER: TRIVIAL + # @PURPOSE: Serves the SPA frontend for any path not matched by API routes. + # @PRE: frontend_path exists. + # @POST: Returns the requested file or index.html. + @app.get("/{file_path:path}", include_in_schema=False) + async def serve_spa(file_path: str): + with belief_scope("serve_spa"): + # Only serve SPA for non-API paths + # API routes are registered separately and should be matched by FastAPI first + if file_path and (file_path.startswith("api/") or file_path.startswith("/api/") or file_path == "api"): + # This should not happen if API routers are properly registered + # Return 404 instead of serving HTML + raise HTTPException(status_code=404, detail=f"API endpoint not found: {file_path}") + + full_path = frontend_path / file_path + if file_path and full_path.is_file(): + return FileResponse(str(full_path)) + return FileResponse(str(frontend_path / "index.html")) + # [/DEF:serve_spa:Function] +else: + # [DEF:read_root:Function] + # @TIER: TRIVIAL + # @PURPOSE: A simple root endpoint to confirm that the API is running when frontend is missing. + # @PRE: None. + # @POST: Returns a JSON message indicating API status. + @app.get("/") + async def read_root(): + with belief_scope("read_root"): + return {"message": "Superset Tools API is running (Frontend build not found)"} + # [/DEF:read_root:Function] +# [/DEF:StaticFiles:Mount] +# [/DEF:AppModule:Module] diff --git a/backend/src/core/async_superset_client.py b/backend/src/core/async_superset_client.py index c674c2d5..64e193c4 100644 --- a/backend/src/core/async_superset_client.py +++ b/backend/src/core/async_superset_client.py @@ -4,8 +4,12 @@ # @SEMANTICS: superset, async, client, httpx, dashboards, datasets # @PURPOSE: Async Superset client for dashboard hot-path requests without blocking FastAPI event loop. # @LAYER: Core -# @RELATION: DEPENDS_ON -> backend.src.core.superset_client -# @RELATION: DEPENDS_ON -> backend.src.core.utils.async_network.AsyncAPIClient +# @PRE: Environment configuration is valid and Superset endpoint is reachable. +# @POST: Provides non-blocking API access to Superset resources. +# @SIDE_EFFECT: Performs network I/O via httpx. +# @DATA_CONTRACT: Input[Environment] -> Model[dashboard, chart, dataset] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.utils.async_network.AsyncAPIClient] # @INVARIANT: Async dashboard operations reuse shared auth cache and avoid sync requests in async routes. # [SECTION: IMPORTS] @@ -22,9 +26,11 @@ from .utils.async_network import AsyncAPIClient # [DEF:AsyncSupersetClient:Class] +# @TIER: STANDARD # @PURPOSE: Async sibling of SupersetClient for dashboard read paths. class AsyncSupersetClient(SupersetClient): # [DEF:__init__:Function] + # @TIER: STANDARD # @PURPOSE: Initialize async Superset client with AsyncAPIClient transport. # @PRE: env is valid. # @POST: Client uses async network transport and inherited projection helpers. @@ -45,6 +51,7 @@ class AsyncSupersetClient(SupersetClient): # [/DEF:__init__:Function] # [DEF:aclose:Function] + # @TIER: STANDARD # @PURPOSE: Close async transport resources. # @POST: Underlying AsyncAPIClient is closed. async def aclose(self) -> None: @@ -52,6 +59,7 @@ class AsyncSupersetClient(SupersetClient): # [/DEF:aclose:Function] # [DEF:get_dashboards_page_async:Function] + # @TIER: STANDARD # @PURPOSE: Fetch one dashboards page asynchronously. # @POST: Returns total count and page result list. async def get_dashboards_page_async(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: @@ -85,6 +93,7 @@ class AsyncSupersetClient(SupersetClient): # [/DEF:get_dashboards_page_async:Function] # [DEF:get_dashboard_async:Function] + # @TIER: STANDARD # @PURPOSE: Fetch one dashboard payload asynchronously. # @POST: Returns raw dashboard payload from Superset API. async def get_dashboard_async(self, dashboard_id: int) -> Dict: @@ -94,6 +103,7 @@ class AsyncSupersetClient(SupersetClient): # [/DEF:get_dashboard_async:Function] # [DEF:get_chart_async:Function] + # @TIER: STANDARD # @PURPOSE: Fetch one chart payload asynchronously. # @POST: Returns raw chart payload from Superset API. async def get_chart_async(self, chart_id: int) -> Dict: @@ -103,6 +113,7 @@ class AsyncSupersetClient(SupersetClient): # [/DEF:get_chart_async:Function] # [DEF:get_dashboard_detail_async:Function] + # @TIER: STANDARD # @PURPOSE: Fetch dashboard detail asynchronously with concurrent charts/datasets requests. # @POST: Returns dashboard detail payload for overview page. async def get_dashboard_detail_async(self, dashboard_id: int) -> Dict: diff --git a/backend/src/core/auth/repository.py b/backend/src/core/auth/repository.py index 77f782e2..bda4a07d 100644 --- a/backend/src/core/auth/repository.py +++ b/backend/src/core/auth/repository.py @@ -4,6 +4,10 @@ # @SEMANTICS: auth, repository, database, user, role, permission # @PURPOSE: Data access layer for authentication and user preference entities. # @LAYER: Domain +# @PRE: SQLAlchemy session manager and auth models are available. +# @POST: Provides transactional access to Auth-related database entities. +# @SIDE_EFFECT: Performs database I/O via SQLAlchemy sessions. +# @DATA_CONTRACT: Input[Session] -> Model[User, Role, Permission, UserDashboardPreference] # @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session] # @RELATION: [DEPENDS_ON] ->[backend.src.models.auth] # @RELATION: [DEPENDS_ON] ->[backend.src.models.profile] @@ -21,10 +25,12 @@ from ..logger import belief_scope, logger # [/SECTION] # [DEF:AuthRepository:Class] +# @TIER: CRITICAL # @PURPOSE: Encapsulates database operations for authentication-related entities. # @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session] class AuthRepository: # [DEF:__init__:Function] + # @TIER: CRITICAL # @PURPOSE: Bind repository instance to an existing SQLAlchemy session. # @PRE: db is an initialized sqlalchemy.orm.Session instance. # @POST: self.db points to the provided session and is used by all repository methods. @@ -42,6 +48,7 @@ class AuthRepository: # [/DEF:__init__:Function] # [DEF:get_user_by_username:Function] + # @TIER: CRITICAL # @PURPOSE: Retrieve a user entity by unique username. # @PRE: username is a non-empty str and self.db is a valid open Session. # @POST: Returns matching User entity when present, otherwise None. @@ -68,6 +75,7 @@ class AuthRepository: # [/DEF:get_user_by_username:Function] # [DEF:get_user_by_id:Function] + # @TIER: CRITICAL # @PURPOSE: Retrieve a user entity by identifier. # @PRE: user_id is a non-empty str and self.db is a valid open Session. # @POST: Returns matching User entity when present, otherwise None. @@ -89,6 +97,7 @@ class AuthRepository: # [/DEF:get_user_by_id:Function] # [DEF:get_role_by_name:Function] + # @TIER: CRITICAL # @PURPOSE: Retrieve a role entity by role name. # @PRE: name is a non-empty str and self.db is a valid open Session. # @POST: Returns matching Role entity when present, otherwise None. @@ -100,6 +109,7 @@ class AuthRepository: # [/DEF:get_role_by_name:Function] # [DEF:update_last_login:Function] + # @TIER: CRITICAL # @PURPOSE: Update last_login timestamp for the provided user entity. # @PRE: user is a managed User instance and self.db is a valid open Session. # @POST: user.last_login is set to current UTC timestamp and transaction is committed. @@ -119,6 +129,7 @@ class AuthRepository: # [/DEF:update_last_login:Function] # [DEF:get_role_by_id:Function] + # @TIER: CRITICAL # @PURPOSE: Retrieve a role entity by identifier. # @PRE: role_id is a non-empty str and self.db is a valid open Session. # @POST: Returns matching Role entity when present, otherwise None. @@ -130,6 +141,7 @@ class AuthRepository: # [/DEF:get_role_by_id:Function] # [DEF:get_permission_by_id:Function] + # @TIER: CRITICAL # @PURPOSE: Retrieve a permission entity by identifier. # @PRE: perm_id is a non-empty str and self.db is a valid open Session. # @POST: Returns matching Permission entity when present, otherwise None. @@ -141,6 +153,7 @@ class AuthRepository: # [/DEF:get_permission_by_id:Function] # [DEF:get_permission_by_resource_action:Function] + # @TIER: CRITICAL # @PURPOSE: Retrieve a permission entity by resource and action pair. # @PRE: resource and action are non-empty str values; self.db is a valid open Session. # @POST: Returns matching Permission entity when present, otherwise None. @@ -155,6 +168,7 @@ class AuthRepository: # [/DEF:get_permission_by_resource_action:Function] # [DEF:get_user_dashboard_preference:Function] + # @TIER: CRITICAL # @PURPOSE: Retrieve dashboard preference entity owned by specified user. # @PRE: user_id is a non-empty str and self.db is a valid open Session. # @POST: Returns matching UserDashboardPreference entity when present, otherwise None. @@ -170,6 +184,7 @@ class AuthRepository: # [/DEF:get_user_dashboard_preference:Function] # [DEF:save_user_dashboard_preference:Function] + # @TIER: CRITICAL # @PURPOSE: Persist dashboard preference entity and return refreshed persistent row. # @PRE: preference is a valid UserDashboardPreference entity and self.db is a valid open Session. # @POST: preference is committed to DB, refreshed from DB state, and returned. @@ -192,6 +207,7 @@ class AuthRepository: # [/DEF:save_user_dashboard_preference:Function] # [DEF:list_permissions:Function] + # @TIER: CRITICAL # @PURPOSE: List all permission entities available in storage. # @PRE: self.db is a valid open Session. # @POST: Returns list containing all Permission entities visible to the session. diff --git a/backend/src/core/config_manager.py b/backend/src/core/config_manager.py index 7a26bc7b..ee214dd1 100644 --- a/backend/src/core/config_manager.py +++ b/backend/src/core/config_manager.py @@ -4,12 +4,15 @@ # @SEMANTICS: config, manager, persistence, migration, postgresql # @PURPOSE: Manages application configuration persistence in DB with one-time migration from legacy JSON. # @LAYER: Domain -# @RELATION: [DEPENDS_ON] ->[ConfigModels] -# @RELATION: [DEPENDS_ON] ->[SessionLocal] -# @RELATION: [DEPENDS_ON] ->[AppConfigRecord] -# @RELATION: [CALLS] ->[logger] -# @RELATION: [CALLS] ->[configure_logger] -# @RELATION: [BINDS_TO] ->[ConfigManager] +# @PRE: Database schema for AppConfigRecord must be initialized. +# @POST: Configuration is loaded into memory and logger is configured. +# @SIDE_EFFECT: Performs DB I/O and may update global logging level. +# @DATA_CONTRACT: Input[json, record] -> Model[AppConfig] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.config_models.AppConfig] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.database.SessionLocal] +# @RELATION: [DEPENDS_ON] ->[backend.src.models.config.AppConfigRecord] +# @RELATION: [CALLS] ->[backend.src.core.logger.logger] +# @RELATION: [CALLS] ->[backend.src.core.logger.configure_logger] # @INVARIANT: Configuration must always be representable by AppConfig and persisted under global record id. # import json @@ -57,6 +60,7 @@ class ConfigManager: # [/DEF:__init__:Function] # [DEF:_default_config:Function] + # @TIER: STANDARD # @PURPOSE: Build default application configuration fallback. # @PRE: None. # @POST: Returns valid AppConfig with empty environments and default storage settings. @@ -71,6 +75,7 @@ class ConfigManager: # [/DEF:_default_config:Function] # [DEF:_sync_raw_payload_from_config:Function] + # @TIER: STANDARD # @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections. # @PRE: self.config is initialized as AppConfig. # @POST: self.raw_payload contains AppConfig fields refreshed from self.config. @@ -85,6 +90,7 @@ class ConfigManager: # [/DEF:_sync_raw_payload_from_config:Function] # [DEF:_load_from_legacy_file:Function] + # @TIER: STANDARD # @PURPOSE: Load legacy JSON configuration for migration fallback path. # @PRE: self.config_path is initialized. # @POST: Returns AppConfig from file payload or safe default. @@ -110,6 +116,7 @@ class ConfigManager: # [/DEF:_load_from_legacy_file:Function] # [DEF:_get_record:Function] + # @TIER: STANDARD # @PURPOSE: Resolve global configuration record from DB. # @PRE: session is an active SQLAlchemy Session. # @POST: Returns record when present, otherwise None. @@ -121,6 +128,7 @@ class ConfigManager: # [/DEF:_get_record:Function] # [DEF:_load_config:Function] + # @TIER: STANDARD # @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON. # @PRE: SessionLocal factory is available and AppConfigRecord schema is accessible. # @POST: Returns valid AppConfig and closes opened DB session. @@ -152,6 +160,7 @@ class ConfigManager: # [/DEF:_load_config:Function] # [DEF:_save_config_to_db:Function] + # @TIER: STANDARD # @PURPOSE: Persist provided AppConfig into the global DB configuration record. # @PRE: config is AppConfig; session is either None or an active Session. # @POST: Global DB record payload equals config.model_dump() when commit succeeds. @@ -186,6 +195,7 @@ class ConfigManager: # [/DEF:_save_config_to_db:Function] # [DEF:save:Function] + # @TIER: STANDARD # @PURPOSE: Persist current in-memory configuration state. # @PRE: self.config is initialized. # @POST: Current self.config is written to DB global record. @@ -197,6 +207,7 @@ class ConfigManager: # [/DEF:save:Function] # [DEF:get_config:Function] + # @TIER: STANDARD # @PURPOSE: Return current in-memory configuration snapshot. # @PRE: self.config is initialized. # @POST: Returns AppConfig reference stored in manager. @@ -208,6 +219,7 @@ class ConfigManager: # [/DEF:get_config:Function] # [DEF:get_payload:Function] + # @TIER: STANDARD # @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema. # @PRE: Manager state is initialized. # @POST: Returns dict payload with current AppConfig fields synchronized. @@ -219,6 +231,7 @@ class ConfigManager: # [/DEF:get_payload:Function] # [DEF:save_config:Function] + # @TIER: STANDARD # @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict. # @PRE: config is AppConfig or dict compatible with AppConfig core schema. # @POST: self.config and self.raw_payload are synchronized and persisted to DB. @@ -240,6 +253,7 @@ class ConfigManager: # [/DEF:save_config:Function] # [DEF:update_global_settings:Function] + # @TIER: STANDARD # @PURPOSE: Replace global settings and persist the resulting configuration. # @PRE: settings is GlobalSettings. # @POST: self.config.settings equals provided settings and DB state is updated. @@ -258,6 +272,7 @@ class ConfigManager: # [/DEF:update_global_settings:Function] # [DEF:validate_path:Function] + # @TIER: STANDARD # @PURPOSE: Validate that path exists and is writable, creating it when absent. # @PRE: path is a string path candidate. # @POST: Returns (True, msg) for writable path, else (False, reason). @@ -279,6 +294,7 @@ class ConfigManager: # [/DEF:validate_path:Function] # [DEF:get_environments:Function] + # @TIER: STANDARD # @PURPOSE: Return all configured environments. # @PRE: self.config is initialized. # @POST: Returns list of Environment models from current configuration. @@ -290,6 +306,7 @@ class ConfigManager: # [/DEF:get_environments:Function] # [DEF:has_environments:Function] + # @TIER: STANDARD # @PURPOSE: Check whether at least one environment exists in configuration. # @PRE: self.config is initialized. # @POST: Returns True iff environment list length is greater than zero. @@ -301,6 +318,7 @@ class ConfigManager: # [/DEF:has_environments:Function] # [DEF:get_environment:Function] + # @TIER: STANDARD # @PURPOSE: Resolve a configured environment by identifier. # @PRE: env_id is string identifier. # @POST: Returns matching Environment when found; otherwise None. @@ -315,6 +333,7 @@ class ConfigManager: # [/DEF:get_environment:Function] # [DEF:add_environment:Function] + # @TIER: STANDARD # @PURPOSE: Upsert environment by id into configuration and persist. # @PRE: env is Environment. # @POST: Configuration contains provided env id with new payload persisted. @@ -333,6 +352,7 @@ class ConfigManager: # [/DEF:add_environment:Function] # [DEF:update_environment:Function] + # @TIER: STANDARD # @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior. # @PRE: env_id is non-empty string and updated_env is Environment. # @POST: Returns True and persists update when target exists; else returns False. @@ -362,6 +382,7 @@ class ConfigManager: # [/DEF:update_environment:Function] # [DEF:delete_environment:Function] + # @TIER: STANDARD # @PURPOSE: Delete environment by id and persist when deletion occurs. # @PRE: env_id is non-empty string. # @POST: Environment is removed when present; otherwise configuration is unchanged. diff --git a/backend/src/core/database.py b/backend/src/core/database.py index ec50ef45..747025fd 100644 --- a/backend/src/core/database.py +++ b/backend/src/core/database.py @@ -4,9 +4,9 @@ # @SEMANTICS: database, postgresql, sqlalchemy, session, persistence # @PURPOSE: Configures database connection and session management (PostgreSQL-first). # @LAYER: Core -# @RELATION: DEPENDS_ON -> sqlalchemy -# @RELATION: DEPENDS_ON -> backend.src.models.mapping -# @RELATION: DEPENDS_ON -> backend.src.core.auth.config +# @RELATION: DEPENDS_ON ->[sqlalchemy] +# @RELATION: DEPENDS_ON ->[backend.src.models.mapping] +# @RELATION: DEPENDS_ON ->[backend.src.core.auth.config] # # @INVARIANT: A single engine instance is used for the entire application. @@ -31,11 +31,13 @@ from pathlib import Path # [/SECTION] # [DEF:BASE_DIR:Variable] +# @TIER: TRIVIAL # @PURPOSE: Base directory for the backend. BASE_DIR = Path(__file__).resolve().parent.parent.parent # [/DEF:BASE_DIR:Variable] # [DEF:DATABASE_URL:Constant] +# @TIER: TRIVIAL # @PURPOSE: URL for the main application database. DEFAULT_POSTGRES_URL = os.getenv( "POSTGRES_URL", @@ -45,34 +47,39 @@ DATABASE_URL = os.getenv("DATABASE_URL", DEFAULT_POSTGRES_URL) # [/DEF:DATABASE_URL:Constant] # [DEF:TASKS_DATABASE_URL:Constant] +# @TIER: TRIVIAL # @PURPOSE: URL for the tasks execution database. # Defaults to DATABASE_URL to keep task logs in the same PostgreSQL instance. TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", DATABASE_URL) # [/DEF:TASKS_DATABASE_URL:Constant] # [DEF:AUTH_DATABASE_URL:Constant] +# @TIER: TRIVIAL # @PURPOSE: URL for the authentication database. AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL) # [/DEF:AUTH_DATABASE_URL:Constant] # [DEF:engine:Variable] +# @TIER: TRIVIAL +# @PURPOSE: SQLAlchemy engine for mappings database. +# @SIDE_EFFECT: Creates database engine and manages connection pool. def _build_engine(db_url: str): with belief_scope("_build_engine"): if db_url.startswith("sqlite"): return create_engine(db_url, connect_args={"check_same_thread": False}) return create_engine(db_url, pool_pre_ping=True) - -# @PURPOSE: SQLAlchemy engine for mappings database. engine = _build_engine(DATABASE_URL) # [/DEF:engine:Variable] # [DEF:tasks_engine:Variable] +# @TIER: TRIVIAL # @PURPOSE: SQLAlchemy engine for tasks database. tasks_engine = _build_engine(TASKS_DATABASE_URL) # [/DEF:tasks_engine:Variable] # [DEF:auth_engine:Variable] +# @TIER: TRIVIAL # @PURPOSE: SQLAlchemy engine for authentication database. auth_engine = _build_engine(AUTH_DATABASE_URL) # [/DEF:auth_engine:Variable] @@ -99,6 +106,7 @@ AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_eng # [/DEF:AuthSessionLocal:Class] # [DEF:_ensure_user_dashboard_preferences_columns:Function] +# @TIER: STANDARD # @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table. # @PRE: bind_engine points to application database where profile table is stored. # @POST: Missing columns are added without data loss. @@ -165,6 +173,7 @@ def _ensure_user_dashboard_preferences_columns(bind_engine): # [DEF:_ensure_user_dashboard_preferences_health_columns:Function] +# @TIER: STANDARD # @PURPOSE: Applies additive schema upgrades for user_dashboard_preferences table (health fields). def _ensure_user_dashboard_preferences_health_columns(bind_engine): with belief_scope("_ensure_user_dashboard_preferences_health_columns"): @@ -208,6 +217,7 @@ def _ensure_user_dashboard_preferences_health_columns(bind_engine): # [DEF:_ensure_llm_validation_results_columns:Function] +# @TIER: STANDARD # @PURPOSE: Applies additive schema upgrades for llm_validation_results table. def _ensure_llm_validation_results_columns(bind_engine): with belief_scope("_ensure_llm_validation_results_columns"): @@ -247,6 +257,7 @@ def _ensure_llm_validation_results_columns(bind_engine): # [DEF:_ensure_git_server_configs_columns:Function] +# @TIER: STANDARD # @PURPOSE: Applies additive schema upgrades for git_server_configs table. # @PRE: bind_engine points to application database. # @POST: Missing columns are added without data loss. @@ -284,6 +295,7 @@ def _ensure_git_server_configs_columns(bind_engine): # [DEF:ensure_connection_configs_table:Function] +# @TIER: STANDARD # @PURPOSE: Ensures the external connection registry table exists in the main database. # @PRE: bind_engine points to the application database. # @POST: connection_configs table exists without dropping existing data. @@ -301,6 +313,7 @@ def ensure_connection_configs_table(bind_engine): # [DEF:init_db:Function] +# @TIER: STANDARD # @PURPOSE: Initializes the database by creating all tables. # @PRE: engine, tasks_engine and auth_engine are initialized. # @POST: Database tables created in all databases. @@ -318,6 +331,7 @@ def init_db(): # [/DEF:init_db:Function] # [DEF:get_db:Function] +# @TIER: STANDARD # @PURPOSE: Dependency for getting a database session. # @PRE: SessionLocal is initialized. # @POST: Session is closed after use. @@ -332,6 +346,7 @@ def get_db(): # [/DEF:get_db:Function] # [DEF:get_tasks_db:Function] +# @TIER: STANDARD # @PURPOSE: Dependency for getting a tasks database session. # @PRE: TasksSessionLocal is initialized. # @POST: Session is closed after use. @@ -346,6 +361,7 @@ def get_tasks_db(): # [/DEF:get_tasks_db:Function] # [DEF:get_auth_db:Function] +# @TIER: STANDARD # @PURPOSE: Dependency for getting an authentication database session. # @PRE: AuthSessionLocal is initialized. # @POST: Session is closed after use. diff --git a/backend/src/core/task_manager/manager.py b/backend/src/core/task_manager/manager.py index 8812d28d..b2b4e31f 100644 --- a/backend/src/core/task_manager/manager.py +++ b/backend/src/core/task_manager/manager.py @@ -3,7 +3,12 @@ # @SEMANTICS: task, manager, lifecycle, execution, state # @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously. # @LAYER: Core -# @RELATION: Depends on PluginLoader to get plugin instances. It is used by the API layer to create and query tasks. +# @PRE: Plugin loader and database sessions are initialized. +# @POST: Orchestrates task execution and persistence. +# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB. +# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.plugin_loader] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.persistence] # @INVARIANT: Task IDs are unique. # @CONSTRAINT: Must use belief_scope for logging. # @TEST_CONTRACT: TaskManagerModule -> { @@ -33,9 +38,9 @@ from ..logger import logger, belief_scope, should_log_task_level # [/SECTION] # [DEF:TaskManager:Class] +# @TIER: CRITICAL # @SEMANTICS: task, manager, lifecycle, execution, state # @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. -# @TIER: CRITICAL # @INVARIANT: Task IDs are unique within the registry. # @INVARIANT: Each task has exactly one status at any time. # @INVARIANT: Log entries are never deleted after being added to a task. @@ -62,6 +67,7 @@ class TaskManager: LOG_FLUSH_INTERVAL = 2.0 # [DEF:__init__:Function] + # @TIER: CRITICAL # @PURPOSE: Initialize the TaskManager with dependencies. # @PRE: plugin_loader is initialized. # @POST: TaskManager is ready to accept tasks. @@ -95,6 +101,7 @@ class TaskManager: # [/DEF:__init__:Function] # [DEF:_flusher_loop:Function] + # @TIER: STANDARD # @PURPOSE: Background thread that periodically flushes log buffer to database. # @PRE: TaskManager is initialized. # @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds. @@ -106,6 +113,7 @@ class TaskManager: # [/DEF:_flusher_loop:Function] # [DEF:_flush_logs:Function] + # @TIER: STANDARD # @PURPOSE: Flush all buffered logs to the database. # @PRE: None. # @POST: All buffered logs are written to task_logs table. @@ -132,6 +140,7 @@ class TaskManager: # [/DEF:_flush_logs:Function] # [DEF:_flush_task_logs:Function] + # @TIER: STANDARD # @PURPOSE: Flush logs for a specific task immediately. # @PRE: task_id exists. # @POST: Task's buffered logs are written to database. @@ -150,6 +159,7 @@ class TaskManager: # [/DEF:_flush_task_logs:Function] # [DEF:create_task:Function] + # @TIER: STANDARD # @PURPOSE: Creates and queues a new task for execution. # @PRE: Plugin with plugin_id exists. Params are valid. # @POST: Task is created, added to registry, and scheduled for execution. @@ -179,6 +189,7 @@ class TaskManager: # [/DEF:create_task:Function] # [DEF:_run_task:Function] + # @TIER: STANDARD # @PURPOSE: Internal method to execute a task with TaskContext support. # @PRE: Task exists in registry. # @POST: Task is executed, status updated to SUCCESS or FAILED. @@ -246,6 +257,7 @@ class TaskManager: # [/DEF:_run_task:Function] # [DEF:resolve_task:Function] + # @TIER: STANDARD # @PURPOSE: Resumes a task that is awaiting mapping. # @PRE: Task exists and is in AWAITING_MAPPING state. # @POST: Task status updated to RUNNING, params updated, execution resumed. @@ -270,6 +282,7 @@ class TaskManager: # [/DEF:resolve_task:Function] # [DEF:wait_for_resolution:Function] + # @TIER: STANDARD # @PURPOSE: Pauses execution and waits for a resolution signal. # @PRE: Task exists. # @POST: Execution pauses until future is set. @@ -292,6 +305,7 @@ class TaskManager: # [/DEF:wait_for_resolution:Function] # [DEF:wait_for_input:Function] + # @TIER: STANDARD # @PURPOSE: Pauses execution and waits for user input. # @PRE: Task exists. # @POST: Execution pauses until future is set via resume_task_with_password. @@ -313,6 +327,7 @@ class TaskManager: # [/DEF:wait_for_input:Function] # [DEF:get_task:Function] + # @TIER: STANDARD # @PURPOSE: Retrieves a task by its ID. # @PRE: task_id is a string. # @POST: Returns Task object or None. @@ -324,6 +339,7 @@ class TaskManager: # [/DEF:get_task:Function] # [DEF:get_all_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Retrieves all registered tasks. # @PRE: None. # @POST: Returns list of all Task objects. @@ -334,6 +350,7 @@ class TaskManager: # [/DEF:get_all_tasks:Function] # [DEF:get_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Retrieves tasks with pagination and optional status filter. # @PRE: limit and offset are non-negative integers. # @POST: Returns a list of tasks sorted by start_time descending. @@ -374,6 +391,7 @@ class TaskManager: # [/DEF:get_tasks:Function] # [DEF:get_task_logs:Function] + # @TIER: STANDARD # @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed). # @PRE: task_id is a string. # @POST: Returns list of LogEntry or TaskLog objects. @@ -406,6 +424,7 @@ class TaskManager: # [/DEF:get_task_logs:Function] # [DEF:get_task_log_stats:Function] + # @TIER: STANDARD # @PURPOSE: Get statistics about logs for a task. # @PRE: task_id is a valid task ID. # @POST: Returns LogStats with counts by level and source. @@ -417,6 +436,7 @@ class TaskManager: # [/DEF:get_task_log_stats:Function] # [DEF:get_task_log_sources:Function] + # @TIER: STANDARD # @PURPOSE: Get unique sources for a task's logs. # @PRE: task_id is a valid task ID. # @POST: Returns list of unique source strings. @@ -428,6 +448,7 @@ class TaskManager: # [/DEF:get_task_log_sources:Function] # [DEF:_add_log:Function] + # @TIER: STANDARD # @PURPOSE: Adds a log entry to a task buffer and notifies subscribers. # @PRE: Task exists. # @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter). @@ -480,6 +501,7 @@ class TaskManager: # [/DEF:_add_log:Function] # [DEF:subscribe_logs:Function] + # @TIER: STANDARD # @PURPOSE: Subscribes to real-time logs for a task. # @PRE: task_id is a string. # @POST: Returns an asyncio.Queue for log entries. @@ -495,6 +517,7 @@ class TaskManager: # [/DEF:subscribe_logs:Function] # [DEF:unsubscribe_logs:Function] + # @TIER: STANDARD # @PURPOSE: Unsubscribes from real-time logs for a task. # @PRE: task_id is a string, queue is asyncio.Queue. # @POST: Queue removed from subscribers. @@ -510,6 +533,7 @@ class TaskManager: # [/DEF:unsubscribe_logs:Function] # [DEF:load_persisted_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Load persisted tasks using persistence service. # @PRE: None. # @POST: Persisted tasks loaded into self.tasks. @@ -522,6 +546,7 @@ class TaskManager: # [/DEF:load_persisted_tasks:Function] # [DEF:await_input:Function] + # @TIER: STANDARD # @PURPOSE: Transition a task to AWAITING_INPUT state with input request. # @PRE: Task exists and is in RUNNING state. # @POST: Task status changed to AWAITING_INPUT, input_request set, persisted. @@ -544,6 +569,7 @@ class TaskManager: # [/DEF:await_input:Function] # [DEF:resume_task_with_password:Function] + # @TIER: STANDARD # @PURPOSE: Resume a task that is awaiting input with provided passwords. # @PRE: Task exists and is in AWAITING_INPUT state. # @POST: Task status changed to RUNNING, passwords injected, task resumed. @@ -573,6 +599,7 @@ class TaskManager: # [/DEF:resume_task_with_password:Function] # [DEF:clear_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Clears tasks based on status filter (also deletes associated logs). # @PRE: status is Optional[TaskStatus]. # @POST: Tasks matching filter (or all non-active) cleared from registry and database. diff --git a/backend/src/core/task_manager/persistence.py b/backend/src/core/task_manager/persistence.py index dfeb1eae..a11a8537 100644 --- a/backend/src/core/task_manager/persistence.py +++ b/backend/src/core/task_manager/persistence.py @@ -3,7 +3,12 @@ # @SEMANTICS: persistence, sqlite, sqlalchemy, task, storage # @PURPOSE: Handles the persistence of tasks using SQLAlchemy and the tasks.db database. # @LAYER: Core -# @RELATION: Used by TaskManager to save and load tasks. +# @PRE: Tasks database must be initialized with TaskRecord and TaskLogRecord schemas. +# @POST: Provides reliable storage and retrieval for task metadata and logs. +# @SIDE_EFFECT: Performs database I/O on tasks.db. +# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord] +# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.database.TasksSessionLocal] # @INVARIANT: Database schema must match the TaskRecord model structure. # [SECTION: IMPORTS] @@ -118,6 +123,7 @@ class TaskPersistenceService: # [/DEF:_resolve_environment_id:Function] # [DEF:__init__:Function] + # @TIER: STANDARD # @PURPOSE: Initializes the persistence service. # @PRE: None. # @POST: Service is ready. @@ -128,6 +134,7 @@ class TaskPersistenceService: # [/DEF:__init__:Function] # [DEF:persist_task:Function] + # @TIER: STANDARD # @PURPOSE: Persists or updates a single task in the database. # @PRE: isinstance(task, Task) # @POST: Task record created or updated in database. @@ -190,6 +197,7 @@ class TaskPersistenceService: # [/DEF:persist_task:Function] # [DEF:persist_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Persists multiple tasks. # @PRE: isinstance(tasks, list) # @POST: All tasks in list are persisted. @@ -201,6 +209,7 @@ class TaskPersistenceService: # [/DEF:persist_tasks:Function] # [DEF:load_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Loads tasks from the database. # @PRE: limit is an integer. # @POST: Returns list of Task objects. @@ -255,6 +264,7 @@ class TaskPersistenceService: # [/DEF:load_tasks:Function] # [DEF:delete_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Deletes specific tasks from the database. # @PRE: task_ids is a list of strings. # @POST: Specified task records deleted from database. @@ -277,9 +287,9 @@ class TaskPersistenceService: # [/DEF:TaskPersistenceService:Class] # [DEF:TaskLogPersistenceService:Class] +# @TIER: CRITICAL # @SEMANTICS: persistence, service, database, log, sqlalchemy # @PURPOSE: Provides methods to save and query task logs from the task_logs table. -# @TIER: CRITICAL # @RELATION: DEPENDS_ON -> TaskLogRecord # @INVARIANT: Log entries are batch-inserted for performance. # @@ -311,6 +321,7 @@ class TaskLogPersistenceService: # [/DEF:__init__:Function] # [DEF:add_logs:Function] + # @TIER: STANDARD # @PURPOSE: Batch insert log entries for a task. # @PRE: logs is a list of LogEntry objects. # @POST: All logs inserted into task_logs table. @@ -342,6 +353,7 @@ class TaskLogPersistenceService: # [/DEF:add_logs:Function] # [DEF:get_logs:Function] + # @TIER: STANDARD # @PURPOSE: Query logs for a task with filtering and pagination. # @PRE: task_id is a valid task ID. # @POST: Returns list of TaskLog objects matching filters. @@ -394,6 +406,7 @@ class TaskLogPersistenceService: # [/DEF:get_logs:Function] # [DEF:get_log_stats:Function] + # @TIER: STANDARD # @PURPOSE: Get statistics about logs for a task. # @PRE: task_id is a valid task ID. # @POST: Returns LogStats with counts by level and source. @@ -439,6 +452,7 @@ class TaskLogPersistenceService: # [/DEF:get_log_stats:Function] # [DEF:get_sources:Function] + # @TIER: STANDARD # @PURPOSE: Get unique sources for a task's logs. # @PRE: task_id is a valid task ID. # @POST: Returns list of unique source strings. @@ -458,6 +472,7 @@ class TaskLogPersistenceService: # [/DEF:get_sources:Function] # [DEF:delete_logs_for_task:Function] + # @TIER: STANDARD # @PURPOSE: Delete all logs for a specific task. # @PRE: task_id is a valid task ID. # @POST: All logs for the task are deleted. @@ -479,6 +494,7 @@ class TaskLogPersistenceService: # [/DEF:delete_logs_for_task:Function] # [DEF:delete_logs_for_tasks:Function] + # @TIER: STANDARD # @PURPOSE: Delete all logs for multiple tasks. # @PRE: task_ids is a list of task IDs. # @POST: All logs for the tasks are deleted. diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index ad2a1ce0..20716bcc 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -1,225 +1,237 @@ -# [DEF:Dependencies:Module] -# @SEMANTICS: dependency, injection, singleton, factory, auth, jwt -# @PURPOSE: Manages creation and provision of shared application dependencies, such as PluginLoader and TaskManager, to avoid circular imports. -# @LAYER: Core -# @RELATION: Used by main app and API routers to get access to shared instances. - -from pathlib import Path -from fastapi import Depends, HTTPException, status -from fastapi.security import OAuth2PasswordBearer -from jose import JWTError -from .core.plugin_loader import PluginLoader -from .core.task_manager import TaskManager -from .core.config_manager import ConfigManager -from .core.scheduler import SchedulerService -from .services.resource_service import ResourceService -from .services.mapping_service import MappingService -from .services.clean_release.repositories import ( - CandidateRepository, ArtifactRepository, ManifestRepository, - PolicyRepository, ComplianceRepository, ReportRepository, - ApprovalRepository, PublicationRepository, AuditRepository, - CleanReleaseAuditLog -) -from .services.clean_release.repository import CleanReleaseRepository -from .services.clean_release.facade import CleanReleaseFacade -from .services.reports.report_service import ReportsService -from .core.database import init_db, get_auth_db, get_db -from .core.logger import logger -from .core.auth.jwt import decode_token -from .core.auth.repository import AuthRepository -from .models.auth import User - -# Initialize singletons -# Use absolute path relative to this file to ensure plugins are found regardless of CWD -project_root = Path(__file__).parent.parent.parent -config_path = project_root / "config.json" - -# Initialize database before services that use persisted configuration. -init_db() -config_manager = ConfigManager(config_path=str(config_path)) - -# [DEF:get_config_manager:Function] -# @PURPOSE: Dependency injector for ConfigManager. -# @PRE: Global config_manager must be initialized. -# @POST: Returns shared ConfigManager instance. -# @RETURN: ConfigManager - The shared config manager instance. -def get_config_manager() -> ConfigManager: - """Dependency injector for ConfigManager.""" - return config_manager -# [/DEF:get_config_manager:Function] - -plugin_dir = Path(__file__).parent / "plugins" - -plugin_loader = PluginLoader(plugin_dir=str(plugin_dir)) -logger.info(f"PluginLoader initialized with directory: {plugin_dir}") -logger.info(f"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}") - -task_manager = TaskManager(plugin_loader) -logger.info("TaskManager initialized") - -scheduler_service = SchedulerService(task_manager, config_manager) -logger.info("SchedulerService initialized") - -resource_service = ResourceService() -logger.info("ResourceService initialized") - -# Clean Release Redesign Singletons -# Note: These use get_db() which is a generator, so we need a way to provide a session. -# For singletons in dependencies.py, we might need a different approach or -# initialize them inside the dependency functions. - -# [DEF:get_plugin_loader:Function] -# @PURPOSE: Dependency injector for PluginLoader. -# @PRE: Global plugin_loader must be initialized. -# @POST: Returns shared PluginLoader instance. -# @RETURN: PluginLoader - The shared plugin loader instance. -def get_plugin_loader() -> PluginLoader: - """Dependency injector for PluginLoader.""" - return plugin_loader -# [/DEF:get_plugin_loader:Function] - -# [DEF:get_task_manager:Function] -# @PURPOSE: Dependency injector for TaskManager. -# @PRE: Global task_manager must be initialized. -# @POST: Returns shared TaskManager instance. -# @RETURN: TaskManager - The shared task manager instance. -def get_task_manager() -> TaskManager: - """Dependency injector for TaskManager.""" - return task_manager -# [/DEF:get_task_manager:Function] - -# [DEF:get_scheduler_service:Function] -# @PURPOSE: Dependency injector for SchedulerService. -# @PRE: Global scheduler_service must be initialized. -# @POST: Returns shared SchedulerService instance. -# @RETURN: SchedulerService - The shared scheduler service instance. -def get_scheduler_service() -> SchedulerService: - """Dependency injector for SchedulerService.""" - return scheduler_service -# [/DEF:get_scheduler_service:Function] - -# [DEF:get_resource_service:Function] -# @PURPOSE: Dependency injector for ResourceService. -# @PRE: Global resource_service must be initialized. -# @POST: Returns shared ResourceService instance. -# @RETURN: ResourceService - The shared resource service instance. -def get_resource_service() -> ResourceService: - """Dependency injector for ResourceService.""" - return resource_service -# [/DEF:get_resource_service:Function] - -# [DEF:get_mapping_service:Function] -# @PURPOSE: Dependency injector for MappingService. -# @PRE: Global config_manager must be initialized. -# @POST: Returns new MappingService instance. -# @RETURN: MappingService - A new mapping service instance. -def get_mapping_service() -> MappingService: - """Dependency injector for MappingService.""" - return MappingService(config_manager) -# [/DEF:get_mapping_service:Function] - - -_clean_release_repository = CleanReleaseRepository() - -# [DEF:get_clean_release_repository:Function] -# @PURPOSE: Legacy compatibility shim for CleanReleaseRepository. -# @POST: Returns a shared CleanReleaseRepository instance. -def get_clean_release_repository() -> CleanReleaseRepository: - """Legacy compatibility shim for CleanReleaseRepository.""" - return _clean_release_repository -# [/DEF:get_clean_release_repository:Function] - - -# [DEF:get_clean_release_facade:Function] -# @PURPOSE: Dependency injector for CleanReleaseFacade. -# @POST: Returns a facade instance with a fresh DB session. -def get_clean_release_facade(db = Depends(get_db)) -> CleanReleaseFacade: - candidate_repo = CandidateRepository(db) - artifact_repo = ArtifactRepository(db) - manifest_repo = ManifestRepository(db) - policy_repo = PolicyRepository(db) - compliance_repo = ComplianceRepository(db) - report_repo = ReportRepository(db) - approval_repo = ApprovalRepository(db) - publication_repo = PublicationRepository(db) - audit_repo = AuditRepository(db) - - return CleanReleaseFacade( - candidate_repo=candidate_repo, - artifact_repo=artifact_repo, - manifest_repo=manifest_repo, - policy_repo=policy_repo, - compliance_repo=compliance_repo, - report_repo=report_repo, - approval_repo=approval_repo, - publication_repo=publication_repo, - audit_repo=audit_repo, - config_manager=config_manager - ) -# [/DEF:get_clean_release_facade:Function] - -# [DEF:oauth2_scheme:Variable] -# @PURPOSE: OAuth2 password bearer scheme for token extraction. -oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login") -# [/DEF:oauth2_scheme:Variable] - -# [DEF:get_current_user:Function] -# @PURPOSE: Dependency for retrieving currently authenticated user from a JWT. -# @PRE: JWT token provided in Authorization header. -# @POST: Returns User object if token is valid. -# @THROW: HTTPException 401 if token is invalid or user not found. -# @PARAM: token (str) - Extracted JWT token. -# @PARAM: db (Session) - Auth database session. -# @RETURN: User - The authenticated user. -def get_current_user(token: str = Depends(oauth2_scheme), db = Depends(get_auth_db)): - credentials_exception = HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Could not validate credentials", - headers={"WWW-Authenticate": "Bearer"}, - ) - try: - payload = decode_token(token) - username: str = payload.get("sub") - if username is None: - raise credentials_exception - except JWTError: - raise credentials_exception - - repo = AuthRepository(db) - user = repo.get_user_by_username(username) - if user is None: - raise credentials_exception - return user -# [/DEF:get_current_user:Function] - -# [DEF:has_permission:Function] -# @PURPOSE: Dependency for checking if the current user has a specific permission. -# @PRE: User is authenticated. -# @POST: Returns True if user has permission. -# @THROW: HTTPException 403 if permission is denied. -# @PARAM: resource (str) - The resource identifier. -# @PARAM: action (str) - The action identifier (READ, EXECUTE, WRITE). -# @RETURN: User - The authenticated user if permission granted. -def has_permission(resource: str, action: str): - def permission_checker(current_user: User = Depends(get_current_user)): - # Union of all permissions across all roles - for role in current_user.roles: - for perm in role.permissions: - if perm.resource == resource and perm.action == action: - return current_user - - # Special case for Admin role (full access) - if any(role.name == "Admin" for role in current_user.roles): - return current_user - - from .core.auth.logger import log_security_event - log_security_event("PERMISSION_DENIED", current_user.username, {"resource": resource, "action": action}) - - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=f"Permission denied for {resource}:{action}" - ) - return permission_checker -# [/DEF:has_permission:Function] - -# [/DEF:Dependencies:Module] +# [DEF:Dependencies:Module] +# @TIER: STANDARD +# @SEMANTICS: dependency, injection, singleton, factory, auth, jwt +# @PURPOSE: Manages creation and provision of shared application dependencies, such as PluginLoader and TaskManager, to avoid circular imports. +# @LAYER: Core +# @RELATION: Used by main app and API routers to get access to shared instances. + +from pathlib import Path +from fastapi import Depends, HTTPException, status +from fastapi.security import OAuth2PasswordBearer +from jose import JWTError +from .core.plugin_loader import PluginLoader +from .core.task_manager import TaskManager +from .core.config_manager import ConfigManager +from .core.scheduler import SchedulerService +from .services.resource_service import ResourceService +from .services.mapping_service import MappingService +from .services.clean_release.repositories import ( + CandidateRepository, ArtifactRepository, ManifestRepository, + PolicyRepository, ComplianceRepository, ReportRepository, + ApprovalRepository, PublicationRepository, AuditRepository, + CleanReleaseAuditLog +) +from .services.clean_release.repository import CleanReleaseRepository +from .services.clean_release.facade import CleanReleaseFacade +from .services.reports.report_service import ReportsService +from .core.database import init_db, get_auth_db, get_db +from .core.logger import logger +from .core.auth.jwt import decode_token +from .core.auth.repository import AuthRepository +from .models.auth import User + +# Initialize singletons +# Use absolute path relative to this file to ensure plugins are found regardless of CWD +project_root = Path(__file__).parent.parent.parent +config_path = project_root / "config.json" + +# Initialize database before services that use persisted configuration. +init_db() +config_manager = ConfigManager(config_path=str(config_path)) + +# [DEF:get_config_manager:Function] +# @TIER: TRIVIAL +# @PURPOSE: Dependency injector for ConfigManager. +# @PRE: Global config_manager must be initialized. +# @POST: Returns shared ConfigManager instance. +# @RETURN: ConfigManager - The shared config manager instance. +def get_config_manager() -> ConfigManager: + """Dependency injector for ConfigManager.""" + return config_manager +# [/DEF:get_config_manager:Function] + +plugin_dir = Path(__file__).parent / "plugins" + +plugin_loader = PluginLoader(plugin_dir=str(plugin_dir)) +logger.info(f"PluginLoader initialized with directory: {plugin_dir}") +logger.info(f"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}") + +task_manager = TaskManager(plugin_loader) +logger.info("TaskManager initialized") + +scheduler_service = SchedulerService(task_manager, config_manager) +logger.info("SchedulerService initialized") + +resource_service = ResourceService() +logger.info("ResourceService initialized") + +# Clean Release Redesign Singletons +# Note: These use get_db() which is a generator, so we need a way to provide a session. +# For singletons in dependencies.py, we might need a different approach or +# initialize them inside the dependency functions. + +# [DEF:get_plugin_loader:Function] +# @TIER: TRIVIAL +# @PURPOSE: Dependency injector for PluginLoader. +# @PRE: Global plugin_loader must be initialized. +# @POST: Returns shared PluginLoader instance. +# @RETURN: PluginLoader - The shared plugin loader instance. +def get_plugin_loader() -> PluginLoader: + """Dependency injector for PluginLoader.""" + return plugin_loader +# [/DEF:get_plugin_loader:Function] + +# [DEF:get_task_manager:Function] +# @TIER: TRIVIAL +# @PURPOSE: Dependency injector for TaskManager. +# @PRE: Global task_manager must be initialized. +# @POST: Returns shared TaskManager instance. +# @RETURN: TaskManager - The shared task manager instance. +def get_task_manager() -> TaskManager: + """Dependency injector for TaskManager.""" + return task_manager +# [/DEF:get_task_manager:Function] + +# [DEF:get_scheduler_service:Function] +# @TIER: TRIVIAL +# @PURPOSE: Dependency injector for SchedulerService. +# @PRE: Global scheduler_service must be initialized. +# @POST: Returns shared SchedulerService instance. +# @RETURN: SchedulerService - The shared scheduler service instance. +def get_scheduler_service() -> SchedulerService: + """Dependency injector for SchedulerService.""" + return scheduler_service +# [/DEF:get_scheduler_service:Function] + +# [DEF:get_resource_service:Function] +# @TIER: TRIVIAL +# @PURPOSE: Dependency injector for ResourceService. +# @PRE: Global resource_service must be initialized. +# @POST: Returns shared ResourceService instance. +# @RETURN: ResourceService - The shared resource service instance. +def get_resource_service() -> ResourceService: + """Dependency injector for ResourceService.""" + return resource_service +# [/DEF:get_resource_service:Function] + +# [DEF:get_mapping_service:Function] +# @TIER: TRIVIAL +# @PURPOSE: Dependency injector for MappingService. +# @PRE: Global config_manager must be initialized. +# @POST: Returns new MappingService instance. +# @RETURN: MappingService - A new mapping service instance. +def get_mapping_service() -> MappingService: + """Dependency injector for MappingService.""" + return MappingService(config_manager) +# [/DEF:get_mapping_service:Function] + + +_clean_release_repository = CleanReleaseRepository() + +# [DEF:get_clean_release_repository:Function] +# @TIER: TRIVIAL +# @PURPOSE: Legacy compatibility shim for CleanReleaseRepository. +# @POST: Returns a shared CleanReleaseRepository instance. +def get_clean_release_repository() -> CleanReleaseRepository: + """Legacy compatibility shim for CleanReleaseRepository.""" + return _clean_release_repository +# [/DEF:get_clean_release_repository:Function] + + +# [DEF:get_clean_release_facade:Function] +# @TIER: TRIVIAL +# @PURPOSE: Dependency injector for CleanReleaseFacade. +# @POST: Returns a facade instance with a fresh DB session. +def get_clean_release_facade(db = Depends(get_db)) -> CleanReleaseFacade: + candidate_repo = CandidateRepository(db) + artifact_repo = ArtifactRepository(db) + manifest_repo = ManifestRepository(db) + policy_repo = PolicyRepository(db) + compliance_repo = ComplianceRepository(db) + report_repo = ReportRepository(db) + approval_repo = ApprovalRepository(db) + publication_repo = PublicationRepository(db) + audit_repo = AuditRepository(db) + + return CleanReleaseFacade( + candidate_repo=candidate_repo, + artifact_repo=artifact_repo, + manifest_repo=manifest_repo, + policy_repo=policy_repo, + compliance_repo=compliance_repo, + report_repo=report_repo, + approval_repo=approval_repo, + publication_repo=publication_repo, + audit_repo=audit_repo, + config_manager=config_manager + ) +# [/DEF:get_clean_release_facade:Function] + +# [DEF:oauth2_scheme:Variable] +# @TIER: TRIVIAL +# @PURPOSE: OAuth2 password bearer scheme for token extraction. +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login") +# [/DEF:oauth2_scheme:Variable] + +# [DEF:get_current_user:Function] +# @TIER: STANDARD +# @PURPOSE: Dependency for retrieving currently authenticated user from a JWT. +# @PRE: JWT token provided in Authorization header. +# @POST: Returns User object if token is valid. +# @THROW: HTTPException 401 if token is invalid or user not found. +# @PARAM: token (str) - Extracted JWT token. +# @PARAM: db (Session) - Auth database session. +# @RETURN: User - The authenticated user. +def get_current_user(token: str = Depends(oauth2_scheme), db = Depends(get_auth_db)): + credentials_exception = HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Could not validate credentials", + headers={"WWW-Authenticate": "Bearer"}, + ) + try: + payload = decode_token(token) + username: str = payload.get("sub") + if username is None: + raise credentials_exception + except JWTError: + raise credentials_exception + + repo = AuthRepository(db) + user = repo.get_user_by_username(username) + if user is None: + raise credentials_exception + return user +# [/DEF:get_current_user:Function] + +# [DEF:has_permission:Function] +# @TIER: STANDARD +# @PURPOSE: Dependency for checking if the current user has a specific permission. +# @PRE: User is authenticated. +# @POST: Returns True if user has permission. +# @THROW: HTTPException 403 if permission is denied. +# @PARAM: resource (str) - The resource identifier. +# @PARAM: action (str) - The action identifier (READ, EXECUTE, WRITE). +# @RETURN: User - The authenticated user if permission granted. +def has_permission(resource: str, action: str): + def permission_checker(current_user: User = Depends(get_current_user)): + # Union of all permissions across all roles + for role in current_user.roles: + for perm in role.permissions: + if perm.resource == resource and perm.action == action: + return current_user + + # Special case for Admin role (full access) + if any(role.name == "Admin" for role in current_user.roles): + return current_user + + from .core.auth.logger import log_security_event + log_security_event("PERMISSION_DENIED", current_user.username, {"resource": resource, "action": action}) + + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Permission denied for {resource}:{action}" + ) + return permission_checker +# [/DEF:has_permission:Function] + +# [/DEF:Dependencies:Module] diff --git a/backend/src/models/clean_release.py b/backend/src/models/clean_release.py index b320e55f..909f4038 100644 --- a/backend/src/models/clean_release.py +++ b/backend/src/models/clean_release.py @@ -3,6 +3,10 @@ # @SEMANTICS: clean-release, models, lifecycle, compliance, evidence, immutability # @PURPOSE: Define canonical clean release domain entities and lifecycle guards. # @LAYER: Domain +# @PRE: Base mapping model and release enums are available. +# @POST: Provides SQLAlchemy and dataclass definitions for governance domain. +# @SIDE_EFFECT: None (schema definition). +# @DATA_CONTRACT: Model[ReleaseCandidate, CandidateArtifact, DistributionManifest, ComplianceRun, ComplianceReport] # @INVARIANT: Immutable snapshots are never mutated; forbidden lifecycle transitions are rejected. from datetime import datetime diff --git a/backend/src/models/report.py b/backend/src/models/report.py index 0ead976a..165f8925 100644 --- a/backend/src/models/report.py +++ b/backend/src/models/report.py @@ -3,7 +3,11 @@ # @SEMANTICS: reports, models, pydantic, normalization, pagination # @PURPOSE: Canonical report schemas for unified task reporting across heterogeneous task types. # @LAYER: Domain -# @RELATION: DEPENDS_ON -> backend.src.core.task_manager.models +# @PRE: Pydantic library and task manager models are available. +# @POST: Provides validated schemas for cross-plugin reporting and UI consumption. +# @SIDE_EFFECT: None (schema definition). +# @DATA_CONTRACT: Model[TaskReport, ReportCollection, ReportDetailView] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.models] # @INVARIANT: Canonical report fields are always present for every report item. # [SECTION: IMPORTS] diff --git a/backend/src/services/auth_service.py b/backend/src/services/auth_service.py index 6a3c95da..511effa0 100644 --- a/backend/src/services/auth_service.py +++ b/backend/src/services/auth_service.py @@ -11,6 +11,10 @@ # @RELATION: [DEPENDS_ON] ->[backend.src.models.auth.Role] # # @INVARIANT: Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles. +# @PRE: Core auth models and security utilities available. +# @POST: User identity verified and session tokens issued according to role scopes. +# @SIDE_EFFECT: Writes last login timestamps and JIT-provisions external users. +# @DATA_CONTRACT: [Credentials | ADFSClaims] -> [UserEntity | SessionToken] # [SECTION: IMPORTS] from typing import Dict, Any @@ -23,9 +27,11 @@ from ..core.logger import belief_scope # [/SECTION] # [DEF:AuthService:Class] +# @TIER: STANDARD # @PURPOSE: Provides high-level authentication services. class AuthService: # [DEF:__init__:Function] + # @TIER: TRIVIAL # @PURPOSE: Initializes the authentication service with repository access over an active DB session. # @PRE: db is a valid SQLAlchemy Session instance bound to the auth persistence context. # @POST: self.repo is initialized and ready for auth user/role CRUD operations. @@ -37,6 +43,7 @@ class AuthService: # [/DEF:__init__:Function] # [DEF:authenticate_user:Function] + # @TIER: STANDARD # @PURPOSE: Validates credentials and account state for local username/password authentication. # @PRE: username and password are non-empty credential inputs. # @POST: Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None. @@ -62,6 +69,7 @@ class AuthService: # [/DEF:authenticate_user:Function] # [DEF:create_session:Function] + # @TIER: STANDARD # @PURPOSE: Issues an access token payload for an already authenticated user. # @PRE: user is a valid User entity containing username and iterable roles with role.name values. # @POST: Returns session dict with non-empty access_token and token_type='bearer'. @@ -87,6 +95,7 @@ class AuthService: # [/DEF:create_session:Function] # [DEF:provision_adfs_user:Function] + # @TIER: STANDARD # @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings. # @PRE: user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent. # @POST: Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state. @@ -121,7 +130,6 @@ class AuthService: self.repo.db.refresh(user) return user # [/DEF:provision_adfs_user:Function] - # [/DEF:AuthService:Class] # [/DEF:backend.src.services.auth_service:Module] \ No newline at end of file diff --git a/backend/src/services/health_service.py b/backend/src/services/health_service.py index 172655d0..a15fd1ac 100644 --- a/backend/src/services/health_service.py +++ b/backend/src/services/health_service.py @@ -3,9 +3,9 @@ # @SEMANTICS: health, aggregation, dashboards # @PURPOSE: Business logic for aggregating dashboard health status from validation records. # @LAYER: Domain/Service -# @RELATION: DEPENDS_ON -> ValidationRecord -# @RELATION: DEPENDS_ON -> backend.src.core.superset_client.SupersetClient -# @RELATION: DEPENDS_ON -> backend.src.core.task_manager.cleanup.TaskCleanupService +# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.cleanup.TaskCleanupService] from typing import List, Dict, Any, Optional, Tuple import time diff --git a/backend/src/services/resource_service.py b/backend/src/services/resource_service.py index a6cf103c..ad313187 100644 --- a/backend/src/services/resource_service.py +++ b/backend/src/services/resource_service.py @@ -18,10 +18,12 @@ from ..core.logger import logger, belief_scope # [/SECTION] # [DEF:ResourceService:Class] +# @TIER: STANDARD # @PURPOSE: Provides centralized access to resource data with enhanced metadata class ResourceService: # [DEF:__init__:Function] + # @TIER: TRIVIAL # @PURPOSE: Initialize the resource service with dependencies # @PRE: None # @POST: ResourceService is ready to fetch resources @@ -32,15 +34,16 @@ class ResourceService: # [/DEF:__init__:Function] # [DEF:get_dashboards_with_status:Function] + # @TIER: STANDARD # @PURPOSE: Fetch dashboards from environment with Git status and last task status # @PRE: env is a valid Environment object # @POST: Returns list of dashboards with enhanced metadata # @PARAM: env (Environment) - The environment to fetch from # @PARAM: tasks (List[Task]) - List of tasks to check for status # @RETURN: List[Dict] - Dashboards with git_status and last_task fields - # @RELATION: CALLS -> SupersetClient.get_dashboards_summary - # @RELATION: CALLS -> self._get_git_status_for_dashboard - # @RELATION: CALLS -> self._get_last_llm_task_for_dashboard + # @RELATION: CALLS ->[SupersetClient:get_dashboards_summary] + # @RELATION: CALLS ->[self:_get_git_status_for_dashboard] + # @RELATION: CALLS ->[self:_get_last_llm_task_for_dashboard] async def get_dashboards_with_status( self, env: Any, @@ -81,6 +84,7 @@ class ResourceService: # [/DEF:get_dashboards_with_status:Function] # [DEF:get_dashboards_page_with_status:Function] + # @TIER: STANDARD # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata. # @PRE: env is valid; page >= 1; page_size > 0. # @POST: Returns page items plus total counters without scanning all pages locally. @@ -144,6 +148,7 @@ class ResourceService: # [/DEF:get_dashboards_page_with_status:Function] # [DEF:_get_last_llm_task_for_dashboard:Function] + # @TIER: STANDARD # @PURPOSE: Get most recent LLM validation task for a dashboard in an environment # @PRE: dashboard_id is a valid integer identifier # @POST: Returns the newest llm_dashboard_validation task summary or None @@ -224,6 +229,7 @@ class ResourceService: # [/DEF:_get_last_llm_task_for_dashboard:Function] # [DEF:_normalize_task_status:Function] + # @TIER: STANDARD # @PURPOSE: Normalize task status to stable uppercase values for UI/API projections # @PRE: raw_status can be enum or string # @POST: Returns uppercase status without enum class prefix @@ -240,6 +246,7 @@ class ResourceService: # [/DEF:_normalize_task_status:Function] # [DEF:_normalize_validation_status:Function] + # @TIER: STANDARD # @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN # @PRE: raw_status can be any scalar type # @POST: Returns normalized validation status token or None @@ -255,6 +262,7 @@ class ResourceService: # [/DEF:_normalize_validation_status:Function] # [DEF:_normalize_datetime_for_compare:Function] + # @TIER: STANDARD # @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons. # @PRE: value may be datetime or any scalar. # @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime. @@ -269,14 +277,15 @@ class ResourceService: # [/DEF:_normalize_datetime_for_compare:Function] # [DEF:get_datasets_with_status:Function] + # @TIER: STANDARD # @PURPOSE: Fetch datasets from environment with mapping progress and last task status # @PRE: env is a valid Environment object # @POST: Returns list of datasets with enhanced metadata # @PARAM: env (Environment) - The environment to fetch from # @PARAM: tasks (List[Task]) - List of tasks to check for status # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields - # @RELATION: CALLS -> SupersetClient.get_datasets_summary - # @RELATION: CALLS -> self._get_last_task_for_resource + # @RELATION: CALLS ->[SupersetClient:get_datasets_summary] + # @RELATION: CALLS ->[self:_get_last_task_for_resource] async def get_datasets_with_status( self, env: Any, @@ -307,6 +316,7 @@ class ResourceService: # [/DEF:get_datasets_with_status:Function] # [DEF:get_activity_summary:Function] + # @TIER: STANDARD # @PURPOSE: Get summary of active and recent tasks for the activity indicator # @PRE: tasks is a list of Task objects # @POST: Returns summary with active_count and recent_tasks @@ -346,12 +356,13 @@ class ResourceService: # [/DEF:get_activity_summary:Function] # [DEF:_get_git_status_for_dashboard:Function] + # @TIER: STANDARD # @PURPOSE: Get Git sync status for a dashboard # @PRE: dashboard_id is a valid integer # @POST: Returns git status or None if no repo exists # @PARAM: dashboard_id (int) - The dashboard ID # @RETURN: Optional[Dict] - Git status with branch and sync_status - # @RELATION: CALLS -> GitService.get_repo + # @RELATION: CALLS ->[GitService:get_repo] def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]: try: repo = self.git_service.get_repo(dashboard_id) @@ -405,6 +416,7 @@ class ResourceService: # [/DEF:_get_git_status_for_dashboard:Function] # [DEF:_get_last_task_for_resource:Function] + # @TIER: STANDARD # @PURPOSE: Get the most recent task for a specific resource # @PRE: resource_id is a valid string # @POST: Returns task summary or None if no tasks found @@ -442,6 +454,7 @@ class ResourceService: # [/DEF:_get_last_task_for_resource:Function] # [DEF:_extract_resource_name_from_task:Function] + # @TIER: STANDARD # @PURPOSE: Extract resource name from task params # @PRE: task is a valid Task object # @POST: Returns resource name or task ID @@ -453,6 +466,7 @@ class ResourceService: # [/DEF:_extract_resource_name_from_task:Function] # [DEF:_extract_resource_type_from_task:Function] + # @TIER: STANDARD # @PURPOSE: Extract resource type from task params # @PRE: task is a valid Task object # @POST: Returns resource type or 'unknown' @@ -462,6 +476,5 @@ class ResourceService: params = task.params or {} return params.get('resource_type', 'unknown') # [/DEF:_extract_resource_type_from_task:Function] - # [/DEF:ResourceService:Class] # [/DEF:backend.src.services.resource_service:Module] diff --git a/specs/020-task-reports-design/tasks.md b/specs/020-task-reports-design/tasks.md index d1a77476..9dde853e 100644 --- a/specs/020-task-reports-design/tasks.md +++ b/specs/020-task-reports-design/tasks.md @@ -130,6 +130,10 @@ - [x] T043 Update semantic map generator to ignore few-shot examples under `.ai/shots/` (`generate_semantic_map.py`) - [x] T044 Fix semantic map generator Svelte relation parsing and prevent path-based CRITICAL over-promotion for nested route entities (`generate_semantic_map.py`) - [x] T045 Preserve latest dashboard validation task identity while falling back only the displayed validation result and broaden Svelte multiline `@RELATION` parsing (`backend/src/services/resource_service.py`, `backend/src/services/__tests__/test_resource_service.py`, `generate_semantic_map.py`) +- [-] T046 Execute `speckit.semantics` workflow with `fix` for semantic remediation + - **Outcome**: A semantic remediation pass was completed, but review gate verdict is **FAIL**. Semantic maintenance advanced the work but did not achieve closure/PASS. + - **Metrics**: Improvements from 1845 -> 1884 contracts and 432 -> 486 relations, with orphans reduced from 1530 -> 1398. Reviewer found current gate metrics at 1884 contracts / 486 relations / 1610 orphans / 301 unresolved relations. + - **Blockers**: CRITICAL metadata incompleteness in `backend/src/core/auth/repository.py`, `backend/src/core/config_manager.py`, `backend/src/core/task_manager/manager.py`, `backend/src/models/report.py`, systemic unresolved relations in `backend/src/api/routes/admin.py` and `backend/src/api/routes/assistant.py`, and missing frontend semantic coverage such as `frontend/src/routes/reports/+page.svelte`. ---