From 9b47b9b6675766228d12c7fd7849412692faf722 Mon Sep 17 00:00:00 2001 From: busya Date: Sat, 21 Mar 2026 15:07:06 +0300 Subject: [PATCH] fix: finalize semantic repair and test updates --- .kilo/agents/closure-gate.md | 2 +- backend/src/api/auth.py | 18 +- .../routes/__tests__/test_assistant_api.py | 31 +- .../routes/__tests__/test_assistant_authz.py | 33 +- .../__tests__/test_clean_release_api.py | 4 + .../test_clean_release_legacy_compat.py | 25 +- .../test_clean_release_source_policy.py | 12 +- .../__tests__/test_clean_release_v2_api.py | 3 + .../test_clean_release_v2_release_api.py | 14 +- .../__tests__/test_connections_routes.py | 6 +- .../src/api/routes/__tests__/test_datasets.py | 159 ++++------ .../src/api/routes/__tests__/test_git_api.py | 44 ++- .../api/routes/__tests__/test_reports_api.py | 14 + .../__tests__/test_reports_detail_api.py | 12 + .../test_reports_openapi_conformance.py | 12 + .../api/routes/__tests__/test_tasks_logs.py | 54 +++- backend/src/api/routes/admin.py | 153 ++++++--- backend/src/api/routes/assistant.py | 59 +++- backend/src/api/routes/connections.py | 57 +++- backend/src/api/routes/dataset_review.py | 292 +++++++++++++++--- backend/src/api/routes/llm.py | 97 ++++-- backend/src/api/routes/reports.py | 47 ++- backend/src/api/routes/settings.py | 11 +- .../__tests__/test_config_manager_compat.py | 27 ++ .../__tests__/test_throttled_scheduler.py | 73 +++-- backend/src/core/auth/config.py | 1 + backend/src/core/auth/jwt.py | 30 +- backend/src/core/auth/oauth.py | 1 + backend/src/core/auth/repository.py | 92 ++++-- backend/src/core/auth/security.py | 1 + backend/src/core/encryption_key.py | 12 +- backend/src/core/mapping_service.py | 187 +++++++---- backend/src/core/migration/__init__.py | 4 +- backend/src/core/migration/archive_parser.py | 14 +- .../core/migration/dry_run_orchestrator.py | 29 +- backend/src/core/migration/risk_assessor.py | 19 +- backend/src/core/scheduler.py | 79 +++-- backend/src/core/task_manager/context.py | 39 ++- backend/src/core/task_manager/manager.py | 242 ++++++++++----- backend/src/core/task_manager/models.py | 56 +++- backend/src/core/task_manager/persistence.py | 17 +- backend/src/core/task_manager/task_logger.py | 90 +++--- backend/src/models/__init__.py | 4 +- backend/src/models/auth.py | 73 +++-- backend/src/scripts/init_auth_db.py | 20 +- backend/src/scripts/seed_permissions.py | 52 +++- .../test_dataset_dashboard_relations.py | 6 + .../__tests__/test_llm_plugin_persistence.py | 48 ++- .../__tests__/test_llm_prompt_templates.py | 30 +- .../services/__tests__/test_llm_provider.py | 76 ++++- backend/src/services/auth_service.py | 48 +-- .../services/dataset_review/orchestrator.py | 17 +- .../__tests__/test_session_repository.py | 169 +++++++--- .../repositories/session_repository.py | 5 +- backend/src/services/health_service.py | 119 +++++-- backend/src/services/mapping_service.py | 44 ++- .../src/services/notifications/providers.py | 21 +- backend/src/services/profile_service.py | 11 +- .../__tests__/test_report_normalizer.py | 8 +- backend/src/services/reports/type_profiles.py | 10 +- .../core/migration/test_archive_parser.py | 17 +- .../migration/test_dry_run_orchestrator.py | 23 +- backend/tests/core/test_migration_engine.py | 39 ++- .../tests/scripts/test_clean_release_cli.py | 35 ++- .../scripts/test_clean_release_tui_v2.py | 11 +- .../test_candidate_manifest_services.py | 95 ++++-- .../test_policy_resolution_service.py | 32 +- backend/tests/test_resource_hubs.py | 184 ++++++++--- frontend/src/components/Navbar.svelte | 2 +- .../StartupEnvironmentWizard.svelte | 2 +- .../src/components/auth/ProtectedRoute.svelte | 6 +- .../src/components/git/CommitModal.svelte | 2 +- .../src/components/llm/ProviderConfig.svelte | 2 +- frontend/src/lib/api/assistant.js | 2 +- .../assistant/AssistantChatPanel.svelte | 4 +- .../lib/components/layout/Breadcrumbs.svelte | 2 +- .../src/lib/components/layout/Sidebar.svelte | 2 +- .../lib/components/layout/TaskDrawer.svelte | 2 +- .../lib/components/layout/TopNavbar.svelte | 2 +- .../lib/components/reports/ReportCard.svelte | 2 +- frontend/src/lib/i18n/index.ts | 4 +- .../lib/stores/__tests__/mocks/navigation.js | 5 + .../src/lib/stores/__tests__/mocks/stores.js | 5 + frontend/src/lib/stores/activity.js | 2 +- frontend/src/lib/stores/health.js | 2 +- frontend/src/routes/admin/roles/+page.svelte | 4 +- .../src/routes/admin/settings/+page.svelte | 4 +- frontend/src/routes/dashboards/+page.svelte | 3 +- .../src/routes/dashboards/[id]/+page.svelte | 2 +- .../src/routes/dashboards/health/+page.svelte | 4 +- frontend/src/routes/datasets/+page.svelte | 3 +- .../src/routes/datasets/[id]/+page.svelte | 2 +- frontend/src/routes/login/+page.svelte | 2 +- .../__tests__/fixtures/profile.fixtures.js | 9 +- frontend/src/routes/reports/+page.svelte | 4 +- frontend/src/routes/settings/+page.svelte | 2 +- .../src/services/__tests__/gitService.test.js | 4 +- frontend/src/services/adminService.js | 4 +- scripts/build_offline_docker_bundle.sh | 5 +- 99 files changed, 2484 insertions(+), 985 deletions(-) diff --git a/.kilo/agents/closure-gate.md b/.kilo/agents/closure-gate.md index f0f39ef5..58859ec3 100644 --- a/.kilo/agents/closure-gate.md +++ b/.kilo/agents/closure-gate.md @@ -5,7 +5,7 @@ model: github-copilot/gpt-5.4-mini temperature: 0.0 permission: edit: deny - bash: deny + bash: allow browser: deny steps: 60 color: primary diff --git a/backend/src/api/auth.py b/backend/src/api/auth.py index 668bb48e..81ca9209 100755 --- a/backend/src/api/auth.py +++ b/backend/src/api/auth.py @@ -4,9 +4,10 @@ # @SEMANTICS: api, auth, routes, login, logout # @PURPOSE: Authentication API endpoints. # @LAYER: API -# @RELATION: USES ->[AuthService:Class] -# @RELATION: USES ->[get_auth_db:Function] -# @RELATION: DEPENDS_ON ->[AuthRepository:Class] +# @RELATION: DEPENDS_ON -> [AuthService] +# @RELATION: DEPENDS_ON -> [get_auth_db] +# @RELATION: DEPENDS_ON -> [get_current_user] +# @RELATION: DEPENDS_ON -> [is_adfs_configured] # @INVARIANT: All auth endpoints must return consistent error codes. # [SECTION: IMPORTS] @@ -24,7 +25,7 @@ import starlette.requests # [/SECTION] # [DEF:router:Variable] -# @RELATION: DEPENDS_ON -> fastapi.APIRouter +# @RELATION: DEPENDS_ON -> [fastapi.APIRouter] # @COMPLEXITY: 1 # @PURPOSE: APIRouter instance for authentication routes. router = APIRouter(prefix="/api/auth", tags=["auth"]) @@ -40,8 +41,8 @@ router = APIRouter(prefix="/api/auth", tags=["auth"]) # @PARAM: form_data (OAuth2PasswordRequestForm) - Login credentials. # @PARAM: db (Session) - Auth database session. # @RETURN: Token - The generated JWT token. -# @RELATION: CALLS -> [authenticate_user] -# @RELATION: CALLS -> [create_session] +# @RELATION: CALLS -> [AuthService.authenticate_user] +# @RELATION: CALLS -> [AuthService.create_session] @router.post("/login", response_model=Token) async def login_for_access_token( form_data: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_auth_db) @@ -125,8 +126,9 @@ async def login_adfs(request: starlette.requests.Request): # @COMPLEXITY: 3 # @PURPOSE: Handles the callback from ADFS after successful authentication. # @POST: Provisions user JIT and returns session token. -# @RELATION: CALLS -> [provision_adfs_user] -# @RELATION: CALLS -> [create_session] +# @RELATION: DEPENDS_ON -> [is_adfs_configured] +# @RELATION: CALLS -> [AuthService.provision_adfs_user] +# @RELATION: CALLS -> [AuthService.create_session] @router.get("/callback/adfs", name="auth_callback_adfs") async def auth_callback_adfs( request: starlette.requests.Request, db: Session = Depends(get_auth_db) diff --git a/backend/src/api/routes/__tests__/test_assistant_api.py b/backend/src/api/routes/__tests__/test_assistant_api.py index b7f94718..4a345af6 100644 --- a/backend/src/api/routes/__tests__/test_assistant_api.py +++ b/backend/src/api/routes/__tests__/test_assistant_api.py @@ -1,4 +1,5 @@ import os + os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w=" # [DEF:AssistantApiTests:Module] # @COMPLEXITY: 3 @@ -23,7 +24,7 @@ from src.models.assistant import AssistantMessageRecord # [DEF:_run_async:Function] -# @RELATION: BINDS_TO -> AssistantApiTests +# @RELATION: BINDS_TO -> [AssistantApiTests] def _run_async(coro): return asyncio.run(coro) @@ -33,6 +34,9 @@ def _run_async(coro): # [DEF:_FakeTask:Class] # @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Lightweight task model stub used as return value from _FakeTaskManager.create_task in assistant route tests. +# @INVARIANT: status is a bare string not a TaskStatus enum; callers must not depend on enum semantics. class _FakeTask: def __init__( self, @@ -94,6 +98,9 @@ class _FakeTaskManager: # [DEF:_FakeConfigManager:Class] # @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 2 +# @PURPOSE: Deterministic config stub providing hardcoded dev/prod environments and minimal settings shape for assistant route tests. +# @INVARIANT: get_config() returns anonymous inner classes, not real GlobalSettings; only default_environment_id and llm fields are safe to access. class _FakeConfigManager: class _Env: def __init__(self, id, name): @@ -119,7 +126,9 @@ class _FakeConfigManager: # [DEF:_admin_user:Function] -# @RELATION: BINDS_TO -> AssistantApiTests +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Build admin principal with spec=User for assistant route authorization tests. def _admin_user(): user = MagicMock(spec=User) user.id = "u-admin" @@ -134,7 +143,9 @@ def _admin_user(): # [DEF:_limited_user:Function] -# @RELATION: BINDS_TO -> AssistantApiTests +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Build limited user principal with empty roles for assistant route denial tests. def _limited_user(): user = MagicMock(spec=User) user.id = "u-limited" @@ -148,6 +159,9 @@ def _limited_user(): # [DEF:_FakeQuery:Class] # @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 2 +# @PURPOSE: Chainable SQLAlchemy-like query stub returning fixed item lists for assistant message persistence paths. +# @INVARIANT: filter() ignores all predicate arguments and returns self; no predicate-based filtering is emulated. class _FakeQuery: def __init__(self, items): self.items = items @@ -183,7 +197,7 @@ class _FakeQuery: # @RELATION: BINDS_TO -> [AssistantApiTests] # @COMPLEXITY: 2 # @PURPOSE: Explicit in-memory DB session double limited to assistant message persistence paths. -# @INVARIANT: query/add/merge stay deterministic and never emulate unrelated SQLAlchemy behavior. +# @INVARIANT: query() always returns _FakeQuery with intentionally non-evaluated predicates; add/merge stay deterministic and never emulate unrelated SQLAlchemy behavior. class _FakeDb: def __init__(self): self.added = [] @@ -213,7 +227,7 @@ class _FakeDb: # [DEF:_clear_assistant_state:Function] -# @RELATION: BINDS_TO -> AssistantApiTests +# @RELATION: BINDS_TO -> [AssistantApiTests] def _clear_assistant_state(): assistant_routes.CONVERSATIONS.clear() assistant_routes.USER_ACTIVE_CONVERSATION.clear() @@ -225,7 +239,7 @@ def _clear_assistant_state(): # [DEF:test_unknown_command_returns_needs_clarification:Function] -# @RELATION: BINDS_TO -> AssistantApiTests +# @RELATION: BINDS_TO -> [AssistantApiTests] # @PURPOSE: Unknown command should return clarification state and unknown intent. def test_unknown_command_returns_needs_clarification(monkeypatch): _clear_assistant_state() @@ -252,7 +266,7 @@ def test_unknown_command_returns_needs_clarification(monkeypatch): # [DEF:test_capabilities_question_returns_successful_help:Function] -# @RELATION: BINDS_TO -> AssistantApiTests +# @RELATION: BINDS_TO -> [AssistantApiTests] # @PURPOSE: Capability query should return deterministic help response. def test_capabilities_question_returns_successful_help(monkeypatch): _clear_assistant_state() @@ -274,7 +288,4 @@ def test_capabilities_question_returns_successful_help(monkeypatch): # [/DEF:test_capabilities_question_returns_successful_help:Function] -# ... (rest of file trimmed for length, I've seen it and I'll keep the existing [DEF]s as is but add @RELATION) -# Note: I'll actually just provide the full file with all @RELATIONs added to reduce orphan count. - # [/DEF:AssistantApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_assistant_authz.py b/backend/src/api/routes/__tests__/test_assistant_authz.py index 9831d737..aee62b8d 100644 --- a/backend/src/api/routes/__tests__/test_assistant_authz.py +++ b/backend/src/api/routes/__tests__/test_assistant_authz.py @@ -1,4 +1,5 @@ import os + os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w=" # [DEF:TestAssistantAuthz:Module] # @COMPLEXITY: 3 @@ -34,7 +35,7 @@ from src.models.assistant import ( # [DEF:_run_async:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Execute async endpoint handler in synchronous test context. # @PRE: coroutine is awaitable endpoint invocation. @@ -47,9 +48,11 @@ def _run_async(coroutine): # [DEF:_FakeTask:Class] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Lightweight task model used for assistant authz tests. +# @PRE: task_id is non-empty string. +# @POST: Returns task with provided id, status, and user_id accessible as attributes. class _FakeTask: def __init__(self, task_id: str, status: str = "RUNNING", user_id: str = "u-admin"): self.id = task_id @@ -59,7 +62,7 @@ class _FakeTask: # [/DEF:_FakeTask:Class] # [DEF:_FakeTaskManager:Class] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 2 # @PURPOSE: In-memory task manager double that records assistant-created tasks deterministically. # @INVARIANT: Only create_task/get_task/get_tasks behavior used by assistant authz routes is emulated. @@ -85,9 +88,12 @@ class _FakeTaskManager: # [/DEF:_FakeTaskManager:Class] # [DEF:_FakeConfigManager:Class] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Provide deterministic environment aliases required by intent parsing. +# @PRE: No external config or DB state is required. +# @POST: get_environments() returns two deterministic SimpleNamespace stubs with id/name. +# @INVARIANT: get_config() is absent; only get_environments() is emulated. Safe only for routes that do not invoke get_config() on the injected ConfigManager — verify against assistant.py route handler code before adding new test cases that use this fake. class _FakeConfigManager: def get_environments(self): return [ @@ -98,7 +104,7 @@ class _FakeConfigManager: # [/DEF:_FakeConfigManager:Class] # [DEF:_admin_user:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Build admin principal fixture. # @PRE: Test requires privileged principal for risky operations. @@ -110,7 +116,7 @@ def _admin_user(): # [/DEF:_admin_user:Function] # [DEF:_other_admin_user:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Build second admin principal fixture for ownership tests. # @PRE: Ownership mismatch scenario needs distinct authenticated actor. @@ -122,7 +128,7 @@ def _other_admin_user(): # [/DEF:_other_admin_user:Function] # [DEF:_limited_user:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Build limited principal without required assistant execution privileges. # @PRE: Permission denial scenario needs non-admin actor. @@ -136,9 +142,10 @@ def _limited_user(): # [DEF:_FakeQuery:Class] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Minimal chainable query object for fake DB interactions. +# @INVARIANT: filter() deliberately discards predicate args and returns self; tests must not assume predicate evaluation. class _FakeQuery: def __init__(self, rows): self._rows = list(rows) @@ -169,7 +176,7 @@ class _FakeQuery: # [/DEF:_FakeQuery:Class] # [DEF:_FakeDb:Class] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 2 # @PURPOSE: In-memory DB session double constrained to assistant message/confirmation/audit persistence paths. # @INVARIANT: query/add/merge are intentionally narrow and must not claim full SQLAlchemy Session semantics. @@ -218,7 +225,7 @@ class _FakeDb: # [/DEF:_FakeDb:Class] # [DEF:_clear_assistant_state:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @COMPLEXITY: 1 # @PURPOSE: Reset assistant process-local state between test cases. # @PRE: Assistant globals may contain state from prior tests. @@ -234,7 +241,7 @@ def _clear_assistant_state(): # [DEF:test_confirmation_owner_mismatch_returns_403:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @PURPOSE: Confirm endpoint should reject requests from user that does not own the confirmation token. # @PRE: Confirmation token is created by first admin actor. # @POST: Second actor receives 403 on confirm operation. @@ -273,7 +280,7 @@ def test_confirmation_owner_mismatch_returns_403(): # [DEF:test_expired_confirmation_cannot_be_confirmed:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @PURPOSE: Expired confirmation token should be rejected and not create task. # @PRE: Confirmation token exists and is manually expired before confirm request. # @POST: Confirm endpoint raises 400 and no task is created. @@ -315,7 +322,7 @@ def test_expired_confirmation_cannot_be_confirmed(): # [DEF:test_limited_user_cannot_launch_restricted_operation:Function] -# @RELATION: BINDS_TO -> TestAssistantAuthz +# @RELATION: BINDS_TO -> [TestAssistantAuthz] # @PURPOSE: Limited user should receive denied state for privileged operation. # @PRE: Restricted user attempts dangerous deploy command. # @POST: Assistant returns denied state and does not execute operation. diff --git a/backend/src/api/routes/__tests__/test_clean_release_api.py b/backend/src/api/routes/__tests__/test_clean_release_api.py index b56dba56..8c0bb70b 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_api.py @@ -79,6 +79,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository: # [DEF:test_start_check_and_get_status_contract:Function] # @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate checks start endpoint returns expected identifiers and status endpoint reflects the same run. def test_start_check_and_get_status_contract(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -116,6 +117,7 @@ def test_start_check_and_get_status_contract(): # [DEF:test_get_report_not_found_returns_404:Function] # @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate reports endpoint returns 404 for an unknown report identifier. def test_get_report_not_found_returns_404(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -132,6 +134,7 @@ def test_get_report_not_found_returns_404(): # [DEF:test_get_report_success:Function] # @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate reports endpoint returns persisted report payload for an existing report identifier. def test_get_report_success(): repo = _repo_with_seed_data() report = ComplianceReport( @@ -161,6 +164,7 @@ def test_get_report_success(): # [DEF:test_prepare_candidate_api_success:Function] # @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate candidate preparation endpoint returns prepared status and manifest identifier on valid input. def test_prepare_candidate_api_success(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo diff --git a/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py b/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py index c9be0d1e..90a02f0e 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py +++ b/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py @@ -12,7 +12,9 @@ from datetime import datetime, timezone from fastapi.testclient import TestClient os.environ.setdefault("DATABASE_URL", "sqlite:///./test_clean_release_legacy_compat.db") -os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///./test_clean_release_legacy_auth.db") +os.environ.setdefault( + "AUTH_DATABASE_URL", "sqlite:///./test_clean_release_legacy_auth.db" +) from src.app import app from src.dependencies import get_clean_release_repository @@ -103,17 +105,23 @@ def _seed_legacy_repo() -> CleanReleaseRepository: created_at=now, created_by="compat-tester", source_snapshot_ref="git:legacy-001", - content_json={"items": [], "summary": {"included_count": 0, "prohibited_detected_count": 0}}, + content_json={ + "items": [], + "summary": {"included_count": 0, "prohibited_detected_count": 0}, + }, immutable=True, ) ) return repo + + # [/DEF:_seed_legacy_repo:Function] # [DEF:test_legacy_prepare_endpoint_still_available:Function] # @RELATION: BINDS_TO -> TestCleanReleaseLegacyCompat +# @PURPOSE: Verify legacy prepare endpoint remains reachable and returns a status payload. def test_legacy_prepare_endpoint_still_available() -> None: repo = _seed_legacy_repo() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -123,7 +131,9 @@ def test_legacy_prepare_endpoint_still_available() -> None: "/api/clean-release/candidates/prepare", json={ "candidate_id": "legacy-rc-001", - "artifacts": [{"path": "src/main.py", "category": "core", "reason": "required"}], + "artifacts": [ + {"path": "src/main.py", "category": "core", "reason": "required"} + ], "sources": ["repo.intra.company.local"], "operator_id": "compat-tester", }, @@ -138,8 +148,10 @@ def test_legacy_prepare_endpoint_still_available() -> None: # [/DEF:test_legacy_prepare_endpoint_still_available:Function] + # [DEF:test_legacy_checks_endpoints_still_available:Function] # @RELATION: BINDS_TO -> TestCleanReleaseLegacyCompat +# @PURPOSE: Verify legacy checks start/status endpoints remain available during v2 transition. def test_legacy_checks_endpoints_still_available() -> None: repo = _seed_legacy_repo() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -159,7 +171,9 @@ def test_legacy_checks_endpoints_still_available() -> None: assert "check_run_id" in start_payload assert start_payload["candidate_id"] == "legacy-rc-001" - status_response = client.get(f"/api/clean-release/checks/{start_payload['check_run_id']}") + status_response = client.get( + f"/api/clean-release/checks/{start_payload['check_run_id']}" + ) assert status_response.status_code == 200 status_payload = status_response.json() assert status_payload["check_run_id"] == start_payload["check_run_id"] @@ -169,4 +183,5 @@ def test_legacy_checks_endpoints_still_available() -> None: app.dependency_overrides.clear() -# [/DEF:TestCleanReleaseLegacyCompat:Module]# [/DEF:test_legacy_checks_endpoints_still_available:Function] +# [/DEF:test_legacy_checks_endpoints_still_available:Function] +# [/DEF:TestCleanReleaseLegacyCompat:Module] diff --git a/backend/src/api/routes/__tests__/test_clean_release_source_policy.py b/backend/src/api/routes/__tests__/test_clean_release_source_policy.py index a0b3f0ee..4544bf6f 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_source_policy.py +++ b/backend/src/api/routes/__tests__/test_clean_release_source_policy.py @@ -24,6 +24,7 @@ from src.services.clean_release.repository import CleanReleaseRepository # [DEF:_repo_with_seed_data:Function] # @RELATION: BINDS_TO -> TestCleanReleaseSourcePolicy +# @PURPOSE: Seed repository with candidate, registry, and active policy for source isolation test flow. def _repo_with_seed_data() -> CleanReleaseRepository: repo = CleanReleaseRepository() @@ -76,8 +77,10 @@ def _repo_with_seed_data() -> CleanReleaseRepository: # [/DEF:_repo_with_seed_data:Function] + # [DEF:test_prepare_candidate_blocks_external_source:Function] # @RELATION: BINDS_TO -> TestCleanReleaseSourcePolicy +# @PURPOSE: Verify candidate preparation is blocked when at least one source host is external to the trusted registry. def test_prepare_candidate_blocks_external_source(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -89,7 +92,11 @@ def test_prepare_candidate_blocks_external_source(): json={ "candidate_id": "2026.03.03-rc1", "artifacts": [ - {"path": "cfg/system.yaml", "category": "system-init", "reason": "required"} + { + "path": "cfg/system.yaml", + "category": "system-init", + "reason": "required", + } ], "sources": ["repo.intra.company.local", "pypi.org"], "operator_id": "release-manager", @@ -103,4 +110,5 @@ def test_prepare_candidate_blocks_external_source(): app.dependency_overrides.clear() -# [/DEF:TestCleanReleaseSourcePolicy:Module]# [/DEF:test_prepare_candidate_blocks_external_source:Function] +# [/DEF:test_prepare_candidate_blocks_external_source:Function] +# [/DEF:TestCleanReleaseSourcePolicy:Module] diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py index 414c8af7..cdd09cde 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py @@ -27,6 +27,7 @@ client = TestClient(app) # [REASON] Implementing API contract tests for candidate/artifact/manifest endpoints (T012). # [DEF:test_candidate_registration_contract:Function] # @RELATION: BINDS_TO -> CleanReleaseV2ApiTests +# @PURPOSE: Validate candidate registration endpoint creates a draft candidate with expected identifier contract. def test_candidate_registration_contract(): """ @TEST_SCENARIO: candidate_registration -> Should return 201 and candidate DTO. @@ -50,6 +51,7 @@ def test_candidate_registration_contract(): # [DEF:test_artifact_import_contract:Function] # @RELATION: BINDS_TO -> CleanReleaseV2ApiTests +# @PURPOSE: Validate artifact import endpoint accepts candidate artifacts and returns success status payload. def test_artifact_import_contract(): """ @TEST_SCENARIO: artifact_import -> Should return 200 and success status. @@ -84,6 +86,7 @@ def test_artifact_import_contract(): # [DEF:test_manifest_build_contract:Function] # @RELATION: BINDS_TO -> CleanReleaseV2ApiTests +# @PURPOSE: Validate manifest build endpoint produces manifest payload linked to the target candidate. def test_manifest_build_contract(): """ @TEST_SCENARIO: manifest_build -> Should return 201 and manifest DTO. diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py index 38141aa6..0790d00c 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py @@ -25,6 +25,7 @@ client = TestClient(test_app) # [DEF:_seed_candidate_and_passed_report:Function] # @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests +# @PURPOSE: Seed repository with approvable candidate and passed report for release endpoint contracts. def _seed_candidate_and_passed_report() -> tuple[str, str]: repository = get_clean_release_repository() candidate_id = f"api-release-candidate-{uuid4()}" @@ -46,7 +47,11 @@ def _seed_candidate_and_passed_report() -> tuple[str, str]: run_id=f"run-{uuid4()}", candidate_id=candidate_id, final_status=ComplianceDecision.PASSED.value, - summary_json={"operator_summary": "ok", "violations_count": 0, "blocking_violations_count": 0}, + summary_json={ + "operator_summary": "ok", + "violations_count": 0, + "blocking_violations_count": 0, + }, generated_at=datetime.now(timezone.utc), immutable=True, ) @@ -56,8 +61,10 @@ def _seed_candidate_and_passed_report() -> tuple[str, str]: # [/DEF:_seed_candidate_and_passed_report:Function] + # [DEF:test_release_approve_and_publish_revoke_contract:Function] # @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests +# @PURPOSE: Verify approve, publish, and revoke endpoints preserve expected release lifecycle contract. def test_release_approve_and_publish_revoke_contract() -> None: """Contract for approve -> publish -> revoke lifecycle endpoints.""" candidate_id, report_id = _seed_candidate_and_passed_report() @@ -98,8 +105,10 @@ def test_release_approve_and_publish_revoke_contract() -> None: # [/DEF:test_release_approve_and_publish_revoke_contract:Function] + # [DEF:test_release_reject_contract:Function] # @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests +# @PURPOSE: Verify reject endpoint returns successful rejection decision payload. def test_release_reject_contract() -> None: """Contract for reject endpoint.""" candidate_id, report_id = _seed_candidate_and_passed_report() @@ -114,4 +123,5 @@ def test_release_reject_contract() -> None: assert payload["decision"] == "REJECTED" -# [/DEF:CleanReleaseV2ReleaseApiTests:Module]# [/DEF:test_release_reject_contract:Function] +# [/DEF:test_release_reject_contract:Function] +# [/DEF:CleanReleaseV2ReleaseApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_connections_routes.py b/backend/src/api/routes/__tests__/test_connections_routes.py index a65fd962..8d024a28 100644 --- a/backend/src/api/routes/__tests__/test_connections_routes.py +++ b/backend/src/api/routes/__tests__/test_connections_routes.py @@ -41,6 +41,7 @@ def db_session(): # [DEF:test_list_connections_bootstraps_missing_table:Function] # @RELATION: BINDS_TO -> ConnectionsRoutesTests +# @PURPOSE: Ensure listing connections auto-creates missing table and returns empty payload. def test_list_connections_bootstraps_missing_table(db_session): from src.api.routes.connections import list_connections @@ -53,8 +54,10 @@ def test_list_connections_bootstraps_missing_table(db_session): # [/DEF:test_list_connections_bootstraps_missing_table:Function] + # [DEF:test_create_connection_bootstraps_missing_table:Function] # @RELATION: BINDS_TO -> ConnectionsRoutesTests +# @PURPOSE: Ensure connection creation bootstraps table and persists returned connection fields. def test_create_connection_bootstraps_missing_table(db_session): from src.api.routes.connections import ConnectionCreate, create_connection @@ -75,5 +78,6 @@ def test_create_connection_bootstraps_missing_table(db_session): assert created.host == "warehouse.internal" assert "connection_configs" in inspector.get_table_names() -# [/DEF:ConnectionsRoutesTests:Module] + # [/DEF:test_create_connection_bootstraps_missing_table:Function] +# [/DEF:ConnectionsRoutesTests:Module] diff --git a/backend/src/api/routes/__tests__/test_datasets.py b/backend/src/api/routes/__tests__/test_datasets.py index ae5bec79..9b616c4a 100644 --- a/backend/src/api/routes/__tests__/test_datasets.py +++ b/backend/src/api/routes/__tests__/test_datasets.py @@ -3,7 +3,7 @@ # @SEMANTICS: datasets, api, tests, pagination, mapping, docs # @PURPOSE: Unit tests for datasets API endpoints. # @LAYER: API -# @RELATION: DEPENDS_ON -> backend.src.api.routes.datasets +# @RELATION: DEPENDS_ON -> [src.api.routes.datasets:Module] # @INVARIANT: Endpoint contracts remain stable for success and validation failure paths. import pytest @@ -11,7 +11,14 @@ from unittest.mock import MagicMock, patch, AsyncMock from fastapi.testclient import TestClient from src.app import app from src.api.routes.datasets import DatasetsResponse, DatasetDetailResponse -from src.dependencies import get_current_user, has_permission, get_config_manager, get_task_manager, get_resource_service, get_mapping_service +from src.dependencies import ( + get_current_user, + has_permission, + get_config_manager, + get_task_manager, + get_resource_service, + get_mapping_service, +) # Global mock user for get_current_user dependency overrides mock_user = MagicMock() @@ -21,49 +28,58 @@ admin_role = MagicMock() admin_role.name = "Admin" mock_user.roles.append(admin_role) + @pytest.fixture(autouse=True) def mock_deps(): + # @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass config_manager = MagicMock() + # @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass task_manager = MagicMock() + # @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass resource_service = MagicMock() mapping_service = MagicMock() - + app.dependency_overrides[get_config_manager] = lambda: config_manager app.dependency_overrides[get_task_manager] = lambda: task_manager app.dependency_overrides[get_resource_service] = lambda: resource_service app.dependency_overrides[get_mapping_service] = lambda: mapping_service app.dependency_overrides[get_current_user] = lambda: mock_user - - app.dependency_overrides[has_permission("plugin:migration", "READ")] = lambda: mock_user - app.dependency_overrides[has_permission("plugin:migration", "EXECUTE")] = lambda: mock_user - app.dependency_overrides[has_permission("plugin:backup", "EXECUTE")] = lambda: mock_user + + app.dependency_overrides[has_permission("plugin:migration", "READ")] = ( + lambda: mock_user + ) + app.dependency_overrides[has_permission("plugin:migration", "EXECUTE")] = ( + lambda: mock_user + ) + app.dependency_overrides[has_permission("plugin:backup", "EXECUTE")] = ( + lambda: mock_user + ) app.dependency_overrides[has_permission("tasks", "READ")] = lambda: mock_user - + yield { "config": config_manager, "task": task_manager, "resource": resource_service, - "mapping": mapping_service + "mapping": mapping_service, } app.dependency_overrides.clear() + client = TestClient(app) # [DEF:test_get_datasets_success:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate successful datasets listing contract for an existing environment. # @TEST: GET /api/datasets returns 200 and valid schema # @PRE: env_id exists # @POST: Response matches DatasetsResponse schema -# [DEF:test_get_datasets_success:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_get_datasets_success(mock_deps): # Mock environment mock_env = MagicMock() mock_env.id = "prod" mock_deps["config"].get_environments.return_value = [mock_env] - + # Mock resource service response mock_deps["resource"].get_datasets_with_status = AsyncMock( return_value=[ @@ -73,13 +89,13 @@ def test_get_datasets_success(mock_deps): "schema": "public", "database": "sales_db", "mapped_fields": {"total": 10, "mapped": 5}, - "last_task": {"task_id": "task-1", "status": "SUCCESS"} + "last_task": {"task_id": "task-1", "status": "SUCCESS"}, } ] ) response = client.get("/api/datasets?env_id=prod") - + assert response.status_code == 200 data = response.json() assert "datasets" in data @@ -92,20 +108,16 @@ def test_get_datasets_success(mock_deps): # [DEF:test_get_datasets_env_not_found:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate datasets listing returns 404 when the requested environment does not exist. # @TEST: GET /api/datasets returns 404 if env_id missing # @PRE: env_id does not exist # @POST: Returns 404 error -# [/DEF:test_get_datasets_success:Function] - -# [DEF:test_get_datasets_env_not_found:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_get_datasets_env_not_found(mock_deps): mock_deps["config"].get_environments.return_value = [] response = client.get("/api/datasets?env_id=nonexistent") - + assert response.status_code == 404 assert "Environment not found" in response.json()["detail"] @@ -114,15 +126,11 @@ def test_get_datasets_env_not_found(mock_deps): # [DEF:test_get_datasets_invalid_pagination:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate datasets listing rejects invalid pagination parameters with 400 responses. # @TEST: GET /api/datasets returns 400 for invalid page/page_size # @PRE: page < 1 or page_size > 100 # @POST: Returns 400 error -# [/DEF:test_get_datasets_env_not_found:Function] - -# [DEF:test_get_datasets_invalid_pagination:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_get_datasets_invalid_pagination(mock_deps): mock_env = MagicMock() mock_env.id = "prod" @@ -132,7 +140,7 @@ def test_get_datasets_invalid_pagination(mock_deps): response = client.get("/api/datasets?env_id=prod&page=0") assert response.status_code == 400 assert "Page must be >= 1" in response.json()["detail"] - + # Invalid page_size (too small) response = client.get("/api/datasets?env_id=prod&page_size=0") assert response.status_code == 400 @@ -148,21 +156,17 @@ def test_get_datasets_invalid_pagination(mock_deps): # [DEF:test_map_columns_success:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate map-columns request creates an async mapping task and returns its identifier. # @TEST: POST /api/datasets/map-columns creates mapping task # @PRE: Valid env_id, dataset_ids, source_type # @POST: Returns task_id -# [/DEF:test_get_datasets_invalid_pagination:Function] - -# [DEF:test_map_columns_success:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_map_columns_success(mock_deps): # Mock environment mock_env = MagicMock() mock_env.id = "prod" mock_deps["config"].get_environments.return_value = [mock_env] - + # Mock task manager mock_task = MagicMock() mock_task.id = "task-123" @@ -170,13 +174,9 @@ def test_map_columns_success(mock_deps): response = client.post( "/api/datasets/map-columns", - json={ - "env_id": "prod", - "dataset_ids": [1, 2, 3], - "source_type": "postgresql" - } + json={"env_id": "prod", "dataset_ids": [1, 2, 3], "source_type": "postgresql"}, ) - + assert response.status_code == 200 data = response.json() assert "task_id" in data @@ -188,25 +188,17 @@ def test_map_columns_success(mock_deps): # [DEF:test_map_columns_invalid_source_type:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate map-columns rejects unsupported source types with a 400 contract response. # @TEST: POST /api/datasets/map-columns returns 400 for invalid source_type # @PRE: source_type is not 'postgresql' or 'xlsx' # @POST: Returns 400 error -# [/DEF:test_map_columns_success:Function] - -# [DEF:test_map_columns_invalid_source_type:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_map_columns_invalid_source_type(mock_deps): response = client.post( "/api/datasets/map-columns", - json={ - "env_id": "prod", - "dataset_ids": [1], - "source_type": "invalid" - } + json={"env_id": "prod", "dataset_ids": [1], "source_type": "invalid"}, ) - + assert response.status_code == 400 assert "Source type must be 'postgresql' or 'xlsx'" in response.json()["detail"] @@ -215,21 +207,17 @@ def test_map_columns_invalid_source_type(mock_deps): # [DEF:test_generate_docs_success:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @TEST: POST /api/datasets/generate-docs creates doc generation task # @PRE: Valid env_id, dataset_ids, llm_provider # @PURPOSE: Validate generate-docs request creates an async documentation task and returns its identifier. # @POST: Returns task_id -# [/DEF:test_map_columns_invalid_source_type:Function] - -# [DEF:test_generate_docs_success:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_generate_docs_success(mock_deps): # Mock environment mock_env = MagicMock() mock_env.id = "prod" mock_deps["config"].get_environments.return_value = [mock_env] - + # Mock task manager mock_task = MagicMock() mock_task.id = "task-456" @@ -237,13 +225,9 @@ def test_generate_docs_success(mock_deps): response = client.post( "/api/datasets/generate-docs", - json={ - "env_id": "prod", - "dataset_ids": [1], - "llm_provider": "openai" - } + json={"env_id": "prod", "dataset_ids": [1], "llm_provider": "openai"}, ) - + assert response.status_code == 200 data = response.json() assert "task_id" in data @@ -255,87 +239,68 @@ def test_generate_docs_success(mock_deps): # [DEF:test_map_columns_empty_ids:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate map-columns rejects empty dataset identifier lists. # @TEST: POST /api/datasets/map-columns returns 400 for empty dataset_ids # @PRE: dataset_ids is empty # @POST: Returns 400 error -# [/DEF:test_generate_docs_success:Function] - -# [DEF:test_map_columns_empty_ids:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_map_columns_empty_ids(mock_deps): """@PRE: dataset_ids must be non-empty.""" response = client.post( "/api/datasets/map-columns", - json={ - "env_id": "prod", - "dataset_ids": [], - "source_type": "postgresql" - } + json={"env_id": "prod", "dataset_ids": [], "source_type": "postgresql"}, ) assert response.status_code == 400 assert "At least one dataset ID must be provided" in response.json()["detail"] + + # [/DEF:test_map_columns_empty_ids:Function] # [DEF:test_generate_docs_empty_ids:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate generate-docs rejects empty dataset identifier lists. # @TEST: POST /api/datasets/generate-docs returns 400 for empty dataset_ids # @PRE: dataset_ids is empty # @POST: Returns 400 error -# [/DEF:test_map_columns_empty_ids:Function] - -# [DEF:test_generate_docs_empty_ids:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_generate_docs_empty_ids(mock_deps): """@PRE: dataset_ids must be non-empty.""" response = client.post( "/api/datasets/generate-docs", - json={ - "env_id": "prod", - "dataset_ids": [], - "llm_provider": "openai" - } + json={"env_id": "prod", "dataset_ids": [], "llm_provider": "openai"}, ) assert response.status_code == 400 assert "At least one dataset ID must be provided" in response.json()["detail"] + + # [/DEF:test_generate_docs_empty_ids:Function] # [DEF:test_generate_docs_env_not_found:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @TEST: POST /api/datasets/generate-docs returns 404 for missing env # @PRE: env_id does not exist # @PURPOSE: Validate generate-docs returns 404 when the requested environment cannot be resolved. # @POST: Returns 404 error -# [/DEF:test_generate_docs_empty_ids:Function] - -# [DEF:test_generate_docs_env_not_found:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests def test_generate_docs_env_not_found(mock_deps): """@PRE: env_id must be a valid environment.""" mock_deps["config"].get_environments.return_value = [] response = client.post( "/api/datasets/generate-docs", - json={ - "env_id": "ghost", - "dataset_ids": [1], - "llm_provider": "openai" - } + json={"env_id": "ghost", "dataset_ids": [1], "llm_provider": "openai"}, ) assert response.status_code == 404 assert "Environment not found" in response.json()["detail"] + + # [/DEF:test_generate_docs_env_not_found:Function] # [DEF:test_get_datasets_superset_failure:Function] -# @RELATION: BINDS_TO -> DatasetsApiTests +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @PURPOSE: Validate datasets listing surfaces a 503 contract when Superset access fails. # @TEST_EDGE: external_superset_failure -> {status: 503} -# [/DEF:test_generate_docs_env_not_found:Function] - +# @POST: Returns 503 with stable error detail when upstream dataset fetch fails. def test_get_datasets_superset_failure(mock_deps): """@TEST_EDGE: external_superset_failure -> {status: 503}""" mock_env = MagicMock() @@ -349,7 +314,9 @@ def test_get_datasets_superset_failure(mock_deps): response = client.get("/api/datasets?env_id=bad_conn") assert response.status_code == 503 assert "Failed to fetch datasets" in response.json()["detail"] + + # [/DEF:test_get_datasets_superset_failure:Function] -# [/DEF:DatasetsApiTests:Module] \ No newline at end of file +# [/DEF:DatasetsApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_git_api.py b/backend/src/api/routes/__tests__/test_git_api.py index 4233c768..e097ab1a 100644 --- a/backend/src/api/routes/__tests__/test_git_api.py +++ b/backend/src/api/routes/__tests__/test_git_api.py @@ -1,6 +1,6 @@ # [DEF:TestGitApi:Module] # @COMPLEXITY: 3 -# @RELATION: VERIFIES ->[src.api.routes.git] +# @RELATION: VERIFIES -> [GitApi] # @PURPOSE: API tests for Git configurations and repository operations. import pytest @@ -12,7 +12,7 @@ from src.models.git import GitServerConfig, GitProvider, GitStatus, GitRepositor # [DEF:DbMock:Class] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] # @COMPLEXITY: 2 # @PURPOSE: In-memory session double for git route tests with minimal query/filter persistence semantics. # @INVARIANT: Supports only the SQLAlchemy-like operations exercised by this test module. @@ -88,7 +88,8 @@ class DbMock: # [DEF:test_get_git_configs_masks_pat:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate listing git configs masks stored PAT values in API-facing responses. def test_get_git_configs_masks_pat(): """ @PRE: Database session `db` is available. @@ -119,7 +120,8 @@ def test_get_git_configs_masks_pat(): # [DEF:test_create_git_config_persists_config:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate creating git config persists supplied server attributes in backing session. def test_create_git_config_persists_config(): """ @PRE: `config` contains valid GitServerConfigCreate data. @@ -153,7 +155,8 @@ from src.api.routes.git_schemas import GitServerConfigUpdate # [DEF:test_update_git_config_modifies_record:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate updating git config modifies mutable fields while preserving masked PAT semantics. def test_update_git_config_modifies_record(): """ @PRE: `config_id` corresponds to an existing configuration. @@ -170,6 +173,7 @@ def test_update_git_config_modifies_record(): ) # The monkeypatched query will return existing_config as it's the only one in the list + # [DEF:SingleConfigDbMock:Class] @PURPOSE: Fake SQLAlchemy session returning single config row. @INVARIANT: Returns hardcoded single-item list; does not simulate empty or multi-row results. class SingleConfigDbMock: def query(self, *args): return self @@ -206,7 +210,8 @@ def test_update_git_config_modifies_record(): # [DEF:test_update_git_config_raises_404_if_not_found:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate updating non-existent git config raises HTTP 404 contract response. def test_update_git_config_raises_404_if_not_found(): """ @PRE: `config_id` corresponds to a missing configuration. @@ -230,7 +235,8 @@ def test_update_git_config_raises_404_if_not_found(): # [DEF:test_delete_git_config_removes_record:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate deleting existing git config removes record and returns success payload. def test_delete_git_config_removes_record(): """ @PRE: `config_id` corresponds to an existing configuration. @@ -238,6 +244,7 @@ def test_delete_git_config_removes_record(): """ existing_config = GitServerConfig(id="config-1") + # [DEF:SingleConfigDbMock:Class] @PURPOSE: Fake SQLAlchemy session returning single config row. @INVARIANT: Returns hardcoded single-item list; does not simulate empty or multi-row results. class SingleConfigDbMock: def query(self, *args): return self @@ -266,13 +273,15 @@ def test_delete_git_config_removes_record(): # [DEF:test_test_git_config_validates_connection_successfully:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate test-connection endpoint returns success when provider connectivity check passes. def test_test_git_config_validates_connection_successfully(monkeypatch): """ @PRE: `config` contains provider, url, and pat. @POST: Returns success if the connection is validated via GitService. """ + # [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths. class MockGitService: async def test_connection(self, provider, url, pat): return True @@ -297,13 +306,15 @@ def test_test_git_config_validates_connection_successfully(monkeypatch): # [DEF:test_test_git_config_fails_validation:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate test-connection endpoint raises HTTP 400 when provider connectivity check fails. def test_test_git_config_fails_validation(monkeypatch): """ @PRE: `config` contains provider, url, and pat BUT connection fails. @THROW: HTTPException 400 """ + # [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths. class MockGitService: async def test_connection(self, provider, url, pat): return False @@ -330,13 +341,15 @@ def test_test_git_config_fails_validation(monkeypatch): # [DEF:test_list_gitea_repositories_returns_payload:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate gitea repositories endpoint returns normalized list for GITEA provider configs. def test_list_gitea_repositories_returns_payload(monkeypatch): """ @PRE: config_id exists and provider is GITEA. @POST: Returns repositories visible to PAT user. """ + # [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths. class MockGitService: async def list_gitea_repositories(self, url, pat): return [ @@ -366,7 +379,8 @@ def test_list_gitea_repositories_returns_payload(monkeypatch): # [DEF:test_list_gitea_repositories_rejects_non_gitea:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate gitea repositories endpoint rejects non-GITEA providers with HTTP 400. def test_list_gitea_repositories_rejects_non_gitea(monkeypatch): """ @PRE: config_id exists and provider is NOT GITEA. @@ -392,13 +406,15 @@ def test_list_gitea_repositories_rejects_non_gitea(monkeypatch): # [DEF:test_create_remote_repository_creates_provider_repo:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate remote repository creation endpoint maps provider response into normalized payload. def test_create_remote_repository_creates_provider_repo(monkeypatch): """ @PRE: config_id exists and PAT has creation permissions. @POST: Returns normalized remote repository payload. """ + # [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths. class MockGitService: async def create_gitlab_repository( self, server_url, pat, name, private, description, auto_init, default_branch @@ -438,7 +454,8 @@ def test_create_remote_repository_creates_provider_repo(monkeypatch): # [DEF:test_init_repository_initializes_and_saves_binding:Function] -# @RELATION: BINDS_TO ->[TestGitApi] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate repository initialization endpoint creates local repo and persists dashboard binding. def test_init_repository_initializes_and_saves_binding(monkeypatch): """ @PRE: `dashboard_ref` exists and `init_data` contains valid config_id and remote_url. @@ -446,6 +463,7 @@ def test_init_repository_initializes_and_saves_binding(monkeypatch): """ from src.api.routes.git_schemas import RepoInitRequest + # [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths. class MockGitService: def init_repo(self, dashboard_id, remote_url, pat, repo_key, default_branch): self.init_called = True diff --git a/backend/src/api/routes/__tests__/test_reports_api.py b/backend/src/api/routes/__tests__/test_reports_api.py index 8d72d073..18e73236 100644 --- a/backend/src/api/routes/__tests__/test_reports_api.py +++ b/backend/src/api/routes/__tests__/test_reports_api.py @@ -16,6 +16,11 @@ from src.core.task_manager.models import Task, TaskStatus from src.dependencies import get_current_user, get_task_manager +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [TestReportsApi] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal task-manager double exposing only get_all_tasks used by reports route tests. +# @INVARIANT: Returns pre-seeded tasks without mutation or side effects. class _FakeTaskManager: def __init__(self, tasks): self._tasks = tasks @@ -24,8 +29,12 @@ class _FakeTaskManager: return self._tasks +# [/DEF:_FakeTaskManager:Class] + + # [DEF:_admin_user:Function] # @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Build deterministic admin principal accepted by reports authorization guard. def _admin_user(): admin_role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(username="test-admin", roles=[admin_role]) @@ -36,6 +45,7 @@ def _admin_user(): # [DEF:_make_task:Function] # @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Build Task fixture with controlled timestamps/status for reports list/detail normalization. def _make_task( task_id: str, plugin_id: str, @@ -60,6 +70,7 @@ def _make_task( # [DEF:test_get_reports_default_pagination_contract:Function] # @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports list endpoint default pagination and contract keys for mixed task statuses. def test_get_reports_default_pagination_contract(): now = datetime.utcnow() tasks = [ @@ -113,6 +124,7 @@ def test_get_reports_default_pagination_contract(): # [DEF:test_get_reports_filter_and_pagination:Function] # @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports list endpoint applies task-type/status filters and pagination boundaries. def test_get_reports_filter_and_pagination(): now = datetime.utcnow() tasks = [ @@ -166,6 +178,7 @@ def test_get_reports_filter_and_pagination(): # [DEF:test_get_reports_handles_mixed_naive_and_aware_datetimes:Function] # @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports sorting remains stable when task timestamps mix naive and timezone-aware datetimes. def test_get_reports_handles_mixed_naive_and_aware_datetimes(): naive_now = datetime.utcnow() aware_now = datetime.now(timezone.utc) @@ -205,6 +218,7 @@ def test_get_reports_handles_mixed_naive_and_aware_datetimes(): # [DEF:test_get_reports_invalid_filter_returns_400:Function] # @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports list endpoint rejects unsupported task type filters with HTTP 400. def test_get_reports_invalid_filter_returns_400(): now = datetime.utcnow() tasks = [ diff --git a/backend/src/api/routes/__tests__/test_reports_detail_api.py b/backend/src/api/routes/__tests__/test_reports_detail_api.py index 66884f04..3c0a82d3 100644 --- a/backend/src/api/routes/__tests__/test_reports_detail_api.py +++ b/backend/src/api/routes/__tests__/test_reports_detail_api.py @@ -16,6 +16,11 @@ from src.core.task_manager.models import Task, TaskStatus from src.dependencies import get_current_user, get_task_manager +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [TestReportsDetailApi] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal task-manager double exposing pre-seeded tasks to detail endpoint under test. +# @INVARIANT: get_all_tasks returns exactly seeded tasks list. class _FakeTaskManager: def __init__(self, tasks): self._tasks = tasks @@ -24,8 +29,12 @@ class _FakeTaskManager: return self._tasks +# [/DEF:_FakeTaskManager:Class] + + # [DEF:_admin_user:Function] # @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Provide admin principal fixture accepted by reports detail authorization policy. def _admin_user(): role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(username="test-admin", roles=[role]) @@ -36,6 +45,7 @@ def _admin_user(): # [DEF:_make_task:Function] # @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Build deterministic Task payload for reports detail endpoint contract assertions. def _make_task(task_id: str, plugin_id: str, status: TaskStatus, result=None): now = datetime.utcnow() return Task( @@ -56,6 +66,7 @@ def _make_task(task_id: str, plugin_id: str, status: TaskStatus, result=None): # [DEF:test_get_report_detail_success:Function] # @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Validate report detail endpoint returns report body with diagnostics and next actions for existing task. def test_get_report_detail_success(): task = _make_task( "detail-1", @@ -91,6 +102,7 @@ def test_get_report_detail_success(): # [DEF:test_get_report_detail_not_found:Function] # @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Validate report detail endpoint returns 404 when requested report identifier is absent. def test_get_report_detail_not_found(): task = _make_task("detail-2", "superset-backup", TaskStatus.SUCCESS) diff --git a/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py b/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py index 5014f190..0b425c0a 100644 --- a/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py +++ b/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py @@ -16,6 +16,11 @@ from src.core.task_manager.models import Task, TaskStatus from src.dependencies import get_current_user, get_task_manager +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [TestReportsOpenapiConformance] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal task-manager fake exposing static task list for OpenAPI conformance checks. +# @INVARIANT: get_all_tasks returns seeded tasks unchanged. class _FakeTaskManager: def __init__(self, tasks): self._tasks = tasks @@ -24,8 +29,12 @@ class _FakeTaskManager: return self._tasks +# [/DEF:_FakeTaskManager:Class] + + # [DEF:_admin_user:Function] # @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Provide admin principal fixture required by reports routes in conformance tests. def _admin_user(): role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(username="test-admin", roles=[role]) @@ -36,6 +45,7 @@ def _admin_user(): # [DEF:_task:Function] # @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Construct deterministic task fixture consumed by reports list/detail payload assertions. def _task(task_id: str, plugin_id: str, status: TaskStatus): now = datetime.utcnow() return Task( @@ -54,6 +64,7 @@ def _task(task_id: str, plugin_id: str, status: TaskStatus): # [DEF:test_reports_list_openapi_required_keys:Function] # @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Verify reports list endpoint includes all required OpenAPI top-level keys. def test_reports_list_openapi_required_keys(): tasks = [ _task("r-1", "superset-backup", TaskStatus.SUCCESS), @@ -86,6 +97,7 @@ def test_reports_list_openapi_required_keys(): # [DEF:test_reports_detail_openapi_required_keys:Function] # @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Verify reports detail endpoint returns payload containing the report object key. def test_reports_detail_openapi_required_keys(): tasks = [_task("r-3", "llm_dashboard_validation", TaskStatus.SUCCESS)] app.dependency_overrides[get_current_user] = lambda: _admin_user() diff --git a/backend/src/api/routes/__tests__/test_tasks_logs.py b/backend/src/api/routes/__tests__/test_tasks_logs.py index 5eed78c6..dd99fda0 100644 --- a/backend/src/api/routes/__tests__/test_tasks_logs.py +++ b/backend/src/api/routes/__tests__/test_tasks_logs.py @@ -1,7 +1,9 @@ -# [DEF:__tests__/test_tasks_logs:Module] -# @RELATION: VERIFIES -> ../tasks.py +# [DEF:test_tasks_logs_module:Module] +# @RELATION: VERIFIES -> [src.api.routes.tasks:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: tests, tasks, logs, api, contract, validation # @PURPOSE: Contract testing for task logs API endpoints. -# [/DEF:__tests__/test_tasks_logs:Module] +# @LAYER: Domain (Tests) import pytest from fastapi import FastAPI @@ -10,35 +12,39 @@ from unittest.mock import MagicMock from src.dependencies import get_task_manager, has_permission from src.api.routes.tasks import router + # @TEST_FIXTURE: mock_app @pytest.fixture def client(): app = FastAPI() app.include_router(router, prefix="/tasks") - + # Mock TaskManager + # @INVARIANT: unconstrained mock — no spec= enforced mock_tm = MagicMock() app.dependency_overrides[get_task_manager] = lambda: mock_tm - + # Mock permissions (bypass for unit test) app.dependency_overrides[has_permission("tasks", "READ")] = lambda: True - + return TestClient(app), mock_tm + # @TEST_CONTRACT: get_task_logs_api -> Invariants # @TEST_FIXTURE: valid_task_logs_request # [DEF:test_get_task_logs_success:Function] -# @RELATION: BINDS_TO -> __tests__/test_tasks_logs +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate task logs endpoint returns filtered logs for an existing task. def test_get_task_logs_success(client): tc, tm = client - + # Setup mock task mock_task = MagicMock() tm.get_task.return_value = mock_task tm.get_task_logs.return_value = [{"level": "INFO", "message": "msg1"}] - + response = tc.get("/tasks/task-1/logs?level=INFO") - + assert response.status_code == 200 assert response.json() == [{"level": "INFO", "message": "msg1"}] tm.get_task.assert_called_with("task-1") @@ -47,42 +53,56 @@ def test_get_task_logs_success(client): assert args[0][0] == "task-1" assert args[0][1].level == "INFO" + # @TEST_EDGE: task_not_found # [/DEF:test_get_task_logs_success:Function] + # [DEF:test_get_task_logs_not_found:Function] -# @RELATION: BINDS_TO -> __tests__/test_tasks_logs +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate task logs endpoint returns 404 when the task identifier is missing. def test_get_task_logs_not_found(client): tc, tm = client tm.get_task.return_value = None - + response = tc.get("/tasks/missing/logs") assert response.status_code == 404 assert response.json()["detail"] == "Task not found" + # @TEST_EDGE: invalid_limit # [/DEF:test_get_task_logs_not_found:Function] + # [DEF:test_get_task_logs_invalid_limit:Function] -# @RELATION: BINDS_TO -> __tests__/test_tasks_logs +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate task logs endpoint enforces query validation for limit lower bound. def test_get_task_logs_invalid_limit(client): tc, tm = client # limit=0 is ge=1 in Query response = tc.get("/tasks/task-1/logs?limit=0") assert response.status_code == 422 + # @TEST_INVARIANT: response_purity # [/DEF:test_get_task_logs_invalid_limit:Function] + # [DEF:test_get_task_log_stats_success:Function] -# @RELATION: BINDS_TO -> __tests__/test_tasks_logs +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate log stats endpoint returns success payload for an existing task. def test_get_task_log_stats_success(client): tc, tm = client tm.get_task.return_value = MagicMock() tm.get_task_log_stats.return_value = {"INFO": 5, "ERROR": 1} - + response = tc.get("/tasks/task-1/logs/stats") assert response.status_code == 200 - # response_model=LogStats might wrap this, but let's check basic structure - # assuming tm.get_task_log_stats returns something compatible with LogStats + + +# response_model=LogStats might wrap this, but let's check basic structure +# assuming tm.get_task_log_stats returns something compatible with LogStats + + # [/DEF:test_get_task_log_stats_success:Function] +# [/DEF:test_tasks_logs_module:Module] diff --git a/backend/src/api/routes/admin.py b/backend/src/api/routes/admin.py index eb70d2c0..78b10db6 100644 --- a/backend/src/api/routes/admin.py +++ b/backend/src/api/routes/admin.py @@ -4,8 +4,9 @@ # @SEMANTICS: api, admin, users, roles, permissions # @PURPOSE: Admin API endpoints for user and role management. # @LAYER: API -# @RELATION: [USES] ->[backend.src.core.auth.repository.AuthRepository] -# @RELATION: [USES] ->[backend.src.dependencies.has_permission] +# @RELATION: [DEPENDS_ON] ->[AuthRepository:Class] +# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function] +# @RELATION: [DEPENDS_ON] ->[has_permission:Function] # # @INVARIANT: All endpoints in this module require 'Admin' role or 'admin' scope. @@ -17,9 +18,15 @@ from ...core.database import get_auth_db from ...core.auth.repository import AuthRepository from ...core.auth.security import get_password_hash from ...schemas.auth import ( - User as UserSchema, UserCreate, UserUpdate, - RoleSchema, RoleCreate, RoleUpdate, PermissionSchema, - ADGroupMappingSchema, ADGroupMappingCreate + User as UserSchema, + UserCreate, + UserUpdate, + RoleSchema, + RoleCreate, + RoleUpdate, + PermissionSchema, + ADGroupMappingSchema, + ADGroupMappingCreate, ) from ...models.auth import User, Role, ADGroupMapping from ...dependencies import has_permission, get_plugin_loader @@ -36,6 +43,7 @@ from ...services.rbac_permission_catalog import ( router = APIRouter(prefix="/api/admin", tags=["admin"]) # [/DEF:router:Variable] + # [DEF:list_users:Function] # @COMPLEXITY: 3 # @PURPOSE: Lists all registered users. @@ -46,14 +54,16 @@ router = APIRouter(prefix="/api/admin", tags=["admin"]) # @RELATION: CALLS -> User @router.get("/users", response_model=List[UserSchema]) async def list_users( - db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:users", "READ")) + db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:users", "READ")) ): with belief_scope("api.admin.list_users"): users = db.query(User).all() return users + + # [/DEF:list_users:Function] + # [DEF:create_user:Function] # @COMPLEXITY: 3 # @PURPOSE: Creates a new local user. @@ -62,37 +72,40 @@ async def list_users( # @PARAM: user_in (UserCreate) - New user data. # @PARAM: db (Session) - Auth database session. # @RETURN: UserSchema - The created user. -# @RELATION: CALLS -> AuthRepository +# @RELATION: [CALLS] ->[AuthRepository:Class] @router.post("/users", response_model=UserSchema, status_code=status.HTTP_201_CREATED) async def create_user( user_in: UserCreate, db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:users", "WRITE")) + _=Depends(has_permission("admin:users", "WRITE")), ): with belief_scope("api.admin.create_user"): repo = AuthRepository(db) if repo.get_user_by_username(user_in.username): raise HTTPException(status_code=400, detail="Username already exists") - + new_user = User( username=user_in.username, email=user_in.email, password_hash=get_password_hash(user_in.password), auth_source="LOCAL", - is_active=user_in.is_active + is_active=user_in.is_active, ) - + for role_name in user_in.roles: role = repo.get_role_by_name(role_name) if role: new_user.roles.append(role) - + db.add(new_user) db.commit() db.refresh(new_user) return new_user + + # [/DEF:create_user:Function] + # [DEF:update_user:Function] # @COMPLEXITY: 3 # @PURPOSE: Updates an existing user. @@ -108,33 +121,36 @@ async def update_user( user_id: str, user_in: UserUpdate, db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:users", "WRITE")) + _=Depends(has_permission("admin:users", "WRITE")), ): with belief_scope("api.admin.update_user"): repo = AuthRepository(db) user = repo.get_user_by_id(user_id) if not user: raise HTTPException(status_code=404, detail="User not found") - + if user_in.email is not None: user.email = user_in.email if user_in.is_active is not None: user.is_active = user_in.is_active if user_in.password is not None: user.password_hash = get_password_hash(user_in.password) - + if user_in.roles is not None: user.roles = [] for role_name in user_in.roles: role = repo.get_role_by_name(role_name) if role: user.roles.append(role) - + db.commit() db.refresh(user) return user + + # [/DEF:update_user:Function] + # [DEF:delete_user:Function] # @COMPLEXITY: 3 # @PURPOSE: Deletes a user. @@ -148,37 +164,50 @@ async def update_user( async def delete_user( user_id: str, db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:users", "WRITE")) + _=Depends(has_permission("admin:users", "WRITE")), ): with belief_scope("api.admin.delete_user"): - logger.info(f"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}") + logger.info( + f"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}" + ) repo = AuthRepository(db) user = repo.get_user_by_id(user_id) if not user: - logger.warning(f"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}") + logger.warning( + f"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}" + ) raise HTTPException(status_code=404, detail="User not found") - - logger.info(f"[DEBUG] Found user to delete context={{'username': '{user.username}'}}") + + logger.info( + f"[DEBUG] Found user to delete context={{'username': '{user.username}'}}" + ) db.delete(user) db.commit() - logger.info(f"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}") + logger.info( + f"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}" + ) return None + + # [/DEF:delete_user:Function] + # [DEF:list_roles:Function] # @COMPLEXITY: 3 # @PURPOSE: Lists all available roles. # @RETURN: List[RoleSchema] - List of roles. -# @RELATION: CALLS -> backend.src.models.auth.Role +# @RELATION: [CALLS] ->[Role:Class] @router.get("/roles", response_model=List[RoleSchema]) async def list_roles( - db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:roles", "READ")) + db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:roles", "READ")) ): with belief_scope("api.admin.list_roles"): return db.query(Role).all() + + # [/DEF:list_roles:Function] + # [DEF:create_role:Function] # @COMPLEXITY: 3 # @PURPOSE: Creates a new system role with associated permissions. @@ -188,35 +217,38 @@ async def list_roles( # @PARAM: db (Session) - Auth database session. # @RETURN: RoleSchema - The created role. # @SIDE_EFFECT: Commits new role and associations to auth.db. -# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.get_permission_by_id +# @RELATION: [CALLS] ->[get_permission_by_id:Function] @router.post("/roles", response_model=RoleSchema, status_code=status.HTTP_201_CREATED) async def create_role( role_in: RoleCreate, db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:roles", "WRITE")) + _=Depends(has_permission("admin:roles", "WRITE")), ): with belief_scope("api.admin.create_role"): if db.query(Role).filter(Role.name == role_in.name).first(): raise HTTPException(status_code=400, detail="Role already exists") - + new_role = Role(name=role_in.name, description=role_in.description) repo = AuthRepository(db) - + for perm_id_or_str in role_in.permissions: perm = repo.get_permission_by_id(perm_id_or_str) if not perm and ":" in perm_id_or_str: res, act = perm_id_or_str.split(":", 1) perm = repo.get_permission_by_resource_action(res, act) - + if perm: new_role.permissions.append(perm) - + db.add(new_role) db.commit() db.refresh(new_role) return new_role + + # [/DEF:create_role:Function] + # [DEF:update_role:Function] # @COMPLEXITY: 3 # @PURPOSE: Updates an existing role's metadata and permissions. @@ -227,25 +259,25 @@ async def create_role( # @PARAM: db (Session) - Auth database session. # @RETURN: RoleSchema - The updated role. # @SIDE_EFFECT: Commits updates to auth.db. -# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.get_role_by_id +# @RELATION: [CALLS] ->[get_role_by_id:Function] @router.put("/roles/{role_id}", response_model=RoleSchema) async def update_role( role_id: str, role_in: RoleUpdate, db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:roles", "WRITE")) + _=Depends(has_permission("admin:roles", "WRITE")), ): with belief_scope("api.admin.update_role"): repo = AuthRepository(db) role = repo.get_role_by_id(role_id) if not role: raise HTTPException(status_code=404, detail="Role not found") - + if role_in.name is not None: role.name = role_in.name if role_in.description is not None: role.description = role_in.description - + if role_in.permissions is not None: role.permissions = [] for perm_id_or_str in role_in.permissions: @@ -253,15 +285,18 @@ async def update_role( if not perm and ":" in perm_id_or_str: res, act = perm_id_or_str.split(":", 1) perm = repo.get_permission_by_resource_action(res, act) - + if perm: role.permissions.append(perm) - + db.commit() db.refresh(role) return role + + # [/DEF:update_role:Function] + # [DEF:delete_role:Function] # @COMPLEXITY: 3 # @PURPOSE: Removes a role from the system. @@ -271,24 +306,27 @@ async def update_role( # @PARAM: db (Session) - Auth database session. # @RETURN: None # @SIDE_EFFECT: Deletes record from auth.db and commits. -# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.get_role_by_id +# @RELATION: [CALLS] ->[get_role_by_id:Function] @router.delete("/roles/{role_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_role( role_id: str, db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:roles", "WRITE")) + _=Depends(has_permission("admin:roles", "WRITE")), ): with belief_scope("api.admin.delete_role"): repo = AuthRepository(db) role = repo.get_role_by_id(role_id) if not role: raise HTTPException(status_code=404, detail="Role not found") - + db.delete(role) db.commit() return None + + # [/DEF:delete_role:Function] + # [DEF:list_permissions:Function] # @COMPLEXITY: 3 # @PURPOSE: Lists all available system permissions for assignment. @@ -299,12 +337,16 @@ async def delete_role( @router.get("/permissions", response_model=List[PermissionSchema]) async def list_permissions( db: Session = Depends(get_auth_db), - plugin_loader = Depends(get_plugin_loader), - _ = Depends(has_permission("admin:roles", "READ")) + plugin_loader=Depends(get_plugin_loader), + _=Depends(has_permission("admin:roles", "READ")), ): with belief_scope("api.admin.list_permissions"): - declared_permissions = discover_declared_permissions(plugin_loader=plugin_loader) - inserted_count = sync_permission_catalog(db=db, declared_permissions=declared_permissions) + declared_permissions = discover_declared_permissions( + plugin_loader=plugin_loader + ) + inserted_count = sync_permission_catalog( + db=db, declared_permissions=declared_permissions + ) if inserted_count > 0: logger.info( "[api.admin.list_permissions][Action] Synchronized %s missing RBAC permissions into auth catalog", @@ -313,8 +355,11 @@ async def list_permissions( repo = AuthRepository(db) return repo.list_permissions() + + # [/DEF:list_permissions:Function] + # [DEF:list_ad_mappings:Function] # @COMPLEXITY: 3 # @PURPOSE: Lists all AD Group to Role mappings. @@ -322,31 +367,37 @@ async def list_permissions( @router.get("/ad-mappings", response_model=List[ADGroupMappingSchema]) async def list_ad_mappings( db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:settings", "READ")) + _=Depends(has_permission("admin:settings", "READ")), ): with belief_scope("api.admin.list_ad_mappings"): return db.query(ADGroupMapping).all() + + # [/DEF:list_ad_mappings:Function] + # [DEF:create_ad_mapping:Function] -# @RELATION: CALLS -> AuthRepository +# @RELATION: [DEPENDS_ON] ->[ADGroupMapping:Class] +# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function] +# @RELATION: [DEPENDS_ON] ->[has_permission:Function] # @COMPLEXITY: 2 # @PURPOSE: Creates a new AD Group mapping. @router.post("/ad-mappings", response_model=ADGroupMappingSchema) async def create_ad_mapping( mapping_in: ADGroupMappingCreate, db: Session = Depends(get_auth_db), - _ = Depends(has_permission("admin:settings", "WRITE")) + _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("api.admin.create_ad_mapping"): new_mapping = ADGroupMapping( - ad_group=mapping_in.ad_group, - role_id=mapping_in.role_id + ad_group=mapping_in.ad_group, role_id=mapping_in.role_id ) db.add(new_mapping) db.commit() db.refresh(new_mapping) return new_mapping + + # [/DEF:create_ad_mapping:Function] -# [/DEF:AdminApi:Module] \ No newline at end of file +# [/DEF:AdminApi:Module] diff --git a/backend/src/api/routes/assistant.py b/backend/src/api/routes/assistant.py index 064d5160..e9f0875b 100644 --- a/backend/src/api/routes/assistant.py +++ b/backend/src/api/routes/assistant.py @@ -3,8 +3,10 @@ # @SEMANTICS: api, assistant, chat, command, confirmation # @PURPOSE: API routes for LLM assistant command parsing and safe execution orchestration. # @LAYER: API -# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.manager.TaskManager] -# @RELATION: [DEPENDS_ON] ->[backend.src.models.assistant] +# @RELATION: DEPENDS_ON -> [TaskManager] +# @RELATION: DEPENDS_ON -> [AssistantMessageRecord] +# @RELATION: DEPENDS_ON -> [AssistantConfirmationRecord] +# @RELATION: DEPENDS_ON -> [AssistantAuditRecord] # @INVARIANT: Risky operations are never executed without valid confirmation token. from __future__ import annotations @@ -54,8 +56,12 @@ git_service = GitService() # [DEF:AssistantMessageRequest:Class] # @COMPLEXITY: 1 # @PURPOSE: Input payload for assistant message endpoint. +# @DATA_CONTRACT: Input[conversation_id?:str, message:str(1..4000)] -> Output[AssistantMessageRequest] +# @RELATION: USED_BY -> [send_message] +# @SIDE_EFFECT: None (schema declaration only). # @PRE: message length is within accepted bounds. # @POST: Request object provides message text and optional conversation binding. +# @INVARIANT: message is always non-empty and no longer than 4000 characters. class AssistantMessageRequest(BaseModel): conversation_id: Optional[str] = None message: str = Field(..., min_length=1, max_length=4000) @@ -67,8 +73,12 @@ class AssistantMessageRequest(BaseModel): # [DEF:AssistantAction:Class] # @COMPLEXITY: 1 # @PURPOSE: UI action descriptor returned with assistant responses. +# @DATA_CONTRACT: Input[type:str, label:str, target?:str] -> Output[AssistantAction] +# @RELATION: USED_BY -> [AssistantMessageResponse] +# @SIDE_EFFECT: None (schema declaration only). # @PRE: type and label are provided by orchestration logic. # @POST: Action can be rendered as button on frontend. +# @INVARIANT: type and label are required for every UI action. class AssistantAction(BaseModel): type: str label: str @@ -81,8 +91,14 @@ class AssistantAction(BaseModel): # [DEF:AssistantMessageResponse:Class] # @COMPLEXITY: 1 # @PURPOSE: Output payload contract for assistant interaction endpoints. +# @DATA_CONTRACT: Input[conversation_id,response_id,state,text,intent?,confirmation_id?,task_id?,actions[],created_at] -> Output[AssistantMessageResponse] +# @RELATION: RETURNED_BY -> [send_message] +# @RELATION: RETURNED_BY -> [confirm_operation] +# @RELATION: RETURNED_BY -> [cancel_operation] +# @SIDE_EFFECT: None (schema declaration only). # @PRE: Response includes deterministic state and text. # @POST: Payload may include task_id/confirmation_id/actions for UI follow-up. +# @INVARIANT: created_at and state are always present in endpoint responses. class AssistantMessageResponse(BaseModel): conversation_id: str response_id: str @@ -101,8 +117,14 @@ class AssistantMessageResponse(BaseModel): # [DEF:ConfirmationRecord:Class] # @COMPLEXITY: 1 # @PURPOSE: In-memory confirmation token model for risky operation dispatch. +# @DATA_CONTRACT: Input[id,user_id,conversation_id,intent,dispatch,expires_at,state?,created_at] -> Output[ConfirmationRecord] +# @RELATION: USED_BY -> [send_message] +# @RELATION: USED_BY -> [confirm_operation] +# @RELATION: USED_BY -> [cancel_operation] +# @SIDE_EFFECT: None (schema declaration only). # @PRE: intent/dispatch/user_id are populated at confirmation request time. # @POST: Record tracks lifecycle state and expiry timestamp. +# @INVARIANT: state defaults to "pending" and expires_at bounds confirmation validity. class ConfirmationRecord(BaseModel): id: str user_id: str @@ -143,8 +165,12 @@ INTENT_PERMISSION_CHECKS: Dict[str, List[Tuple[str, str]]] = { # [DEF:_append_history:Function] # @COMPLEXITY: 2 # @PURPOSE: Append conversation message to in-memory history buffer. +# @DATA_CONTRACT: Input[user_id,conversation_id,role,text,state?,task_id?,confirmation_id?] -> Output[None] +# @RELATION: UPDATES -> [CONVERSATIONS] +# @SIDE_EFFECT: Mutates in-memory CONVERSATIONS store for user conversation history. # @PRE: user_id and conversation_id identify target conversation bucket. # @POST: Message entry is appended to CONVERSATIONS key list. +# @INVARIANT: every appended entry includes generated message_id and created_at timestamp. def _append_history( user_id: str, conversation_id: str, @@ -177,8 +203,12 @@ def _append_history( # [DEF:_persist_message:Function] # @COMPLEXITY: 2 # @PURPOSE: Persist assistant/user message record to database. +# @DATA_CONTRACT: Input[Session,user_id,conversation_id,role,text,state?,task_id?,confirmation_id?,metadata?] -> Output[None] +# @RELATION: DEPENDS_ON -> [AssistantMessageRecord] +# @SIDE_EFFECT: Writes AssistantMessageRecord rows and commits or rollbacks the DB session. # @PRE: db session is writable and message payload is serializable. # @POST: Message row is committed or persistence failure is logged. +# @INVARIANT: failed persistence attempts always rollback before returning. def _persist_message( db: Session, user_id: str, @@ -215,8 +245,12 @@ def _persist_message( # [DEF:_audit:Function] # @COMPLEXITY: 2 # @PURPOSE: Append in-memory audit record for assistant decision trace. +# @DATA_CONTRACT: Input[user_id,payload:Dict[str,Any]] -> Output[None] +# @RELATION: UPDATES -> [ASSISTANT_AUDIT] +# @SIDE_EFFECT: Mutates in-memory ASSISTANT_AUDIT store and emits structured log event. # @PRE: payload describes decision/outcome fields. # @POST: ASSISTANT_AUDIT list for user contains new timestamped entry. +# @INVARIANT: persisted in-memory audit entry always contains created_at in ISO format. def _audit(user_id: str, payload: Dict[str, Any]): if user_id not in ASSISTANT_AUDIT: ASSISTANT_AUDIT[user_id] = [] @@ -852,8 +886,13 @@ def _build_task_observability_summary(task: Any, config_manager: ConfigManager) # [DEF:_parse_command:Function] # @COMPLEXITY: 4 # @PURPOSE: Deterministically parse RU/EN command text into intent payload. +# @DATA_CONTRACT: Input[message:str, config_manager:ConfigManager] -> Output[Dict[str,Any]{domain,operation,entities,confidence,risk_level,requires_confirmation}] +# @RELATION: DEPENDS_ON -> [_extract_id] +# @RELATION: DEPENDS_ON -> [_is_production_env] +# @SIDE_EFFECT: None (pure parsing logic). # @PRE: message contains raw user text and config manager resolves environments. # @POST: Returns intent dict with domain/operation/entities/confidence/risk fields. +# @INVARIANT: every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation. def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any]: text = message.strip() lower = text.lower() @@ -1575,8 +1614,15 @@ def _authorize_intent(intent: Dict[str, Any], current_user: User): # [DEF:_dispatch_intent:Function] # @COMPLEXITY: 5 # @PURPOSE: Execute parsed assistant intent via existing task/plugin/git services. +# @DATA_CONTRACT: Input[intent,current_user,task_manager,config_manager,db] -> Output[Tuple[text:str,task_id:Optional[str],actions:List[AssistantAction]]] +# @RELATION: DEPENDS_ON -> [_check_any_permission] +# @RELATION: DEPENDS_ON -> [_resolve_dashboard_id_entity] +# @RELATION: DEPENDS_ON -> [TaskManager] +# @RELATION: DEPENDS_ON -> [GitService] +# @SIDE_EFFECT: May enqueue tasks, invoke git operations, and query/update external service state. # @PRE: intent operation is known and actor permissions are validated per operation. # @POST: Returns response text, optional task id, and UI actions for follow-up. +# @INVARIANT: unsupported operations are rejected via HTTPException(400). async def _dispatch_intent( intent: Dict[str, Any], current_user: User, @@ -1974,9 +2020,18 @@ async def _dispatch_intent( # [DEF:send_message:Function] # @COMPLEXITY: 5 # @PURPOSE: Parse assistant command, enforce safety gates, and dispatch executable intent. +# @DATA_CONTRACT: Input[AssistantMessageRequest,User,TaskManager,ConfigManager,Session] -> Output[AssistantMessageResponse] +# @RELATION: DEPENDS_ON -> [_plan_intent_with_llm] +# @RELATION: DEPENDS_ON -> [_parse_command] +# @RELATION: DEPENDS_ON -> [_dispatch_intent] +# @RELATION: DEPENDS_ON -> [_append_history] +# @RELATION: DEPENDS_ON -> [_persist_message] +# @RELATION: DEPENDS_ON -> [_audit] +# @SIDE_EFFECT: Persists chat/audit state, mutates in-memory conversation and confirmation stores, and may create confirmation records. # @PRE: Authenticated user is available and message text is non-empty. # @POST: Response state is one of clarification/confirmation/started/success/denied/failed. # @RETURN: AssistantMessageResponse with operation feedback and optional actions. +# @INVARIANT: non-safe operations are gated with confirmation before execution from this endpoint. async def send_message( request: AssistantMessageRequest, current_user: User = Depends(get_current_user), diff --git a/backend/src/api/routes/connections.py b/backend/src/api/routes/connections.py index 012fe1a8..f51c745f 100644 --- a/backend/src/api/routes/connections.py +++ b/backend/src/api/routes/connections.py @@ -1,8 +1,9 @@ # [DEF:ConnectionsRouter:Module] # @SEMANTICS: api, router, connections, database # @PURPOSE: Defines the FastAPI router for managing external database connections. +# @COMPLEXITY: 3 # @LAYER: UI (API) -# @RELATION: Depends on SQLAlchemy session. +# @RELATION: DEPENDS_ON -> Session # @CONSTRAINT: Must use belief_scope for logging. # [SECTION: IMPORTS] @@ -21,15 +22,22 @@ router = APIRouter() # [DEF:_ensure_connections_schema:Function] # @PURPOSE: Ensures the connection_configs table exists before CRUD access. +# @COMPLEXITY: 3 # @PRE: db is an active SQLAlchemy session. # @POST: The current bind can safely query ConnectionConfig. +# @RELATION: CALLS -> ensure_connection_configs_table def _ensure_connections_schema(db: Session): with belief_scope("ConnectionsRouter.ensure_schema"): ensure_connection_configs_table(db.get_bind()) + + # [/DEF:_ensure_connections_schema:Function] + # [DEF:ConnectionSchema:Class] # @PURPOSE: Pydantic model for connection response. +# @COMPLEXITY: 3 +# @RELATION: BINDS_TO -> ConnectionConfig class ConnectionSchema(BaseModel): id: str name: str @@ -42,10 +50,15 @@ class ConnectionSchema(BaseModel): class Config: orm_mode = True + + # [/DEF:ConnectionSchema:Class] + # [DEF:ConnectionCreate:Class] # @PURPOSE: Pydantic model for creating a connection. +# @COMPLEXITY: 3 +# @RELATION: BINDS_TO -> ConnectionConfig class ConnectionCreate(BaseModel): name: str type: str @@ -54,60 +67,92 @@ class ConnectionCreate(BaseModel): database: Optional[str] = None username: Optional[str] = None password: Optional[str] = None + + # [/DEF:ConnectionCreate:Class] + # [DEF:list_connections:Function] # @PURPOSE: Lists all saved connections. +# @COMPLEXITY: 3 # @PRE: Database session is active. # @POST: Returns list of connection configs. # @PARAM: db (Session) - Database session. # @RETURN: List[ConnectionSchema] - List of connections. +# @RELATION: CALLS -> _ensure_connections_schema +# @RELATION: DEPENDS_ON -> ConnectionConfig @router.get("", response_model=List[ConnectionSchema]) async def list_connections(db: Session = Depends(get_db)): with belief_scope("ConnectionsRouter.list_connections"): _ensure_connections_schema(db) connections = db.query(ConnectionConfig).all() return connections + + # [/DEF:list_connections:Function] + # [DEF:create_connection:Function] # @PURPOSE: Creates a new connection configuration. +# @COMPLEXITY: 3 # @PRE: Connection name is unique. # @POST: Connection is saved to DB. # @PARAM: connection (ConnectionCreate) - Config data. # @PARAM: db (Session) - Database session. # @RETURN: ConnectionSchema - Created connection. +# @RELATION: CALLS -> _ensure_connections_schema +# @RELATION: DEPENDS_ON -> ConnectionConfig @router.post("", response_model=ConnectionSchema, status_code=status.HTTP_201_CREATED) -async def create_connection(connection: ConnectionCreate, db: Session = Depends(get_db)): +async def create_connection( + connection: ConnectionCreate, db: Session = Depends(get_db) +): with belief_scope("ConnectionsRouter.create_connection", f"name={connection.name}"): _ensure_connections_schema(db) db_connection = ConnectionConfig(**connection.dict()) db.add(db_connection) db.commit() db.refresh(db_connection) - logger.info(f"[ConnectionsRouter.create_connection][Success] Created connection {db_connection.id}") + logger.info( + f"[ConnectionsRouter.create_connection][Success] Created connection {db_connection.id}" + ) return db_connection + + # [/DEF:create_connection:Function] + # [DEF:delete_connection:Function] # @PURPOSE: Deletes a connection configuration. +# @COMPLEXITY: 3 # @PRE: Connection ID exists. # @POST: Connection is removed from DB. # @PARAM: connection_id (str) - ID to delete. # @PARAM: db (Session) - Database session. # @RETURN: None. +# @RELATION: CALLS -> _ensure_connections_schema +# @RELATION: DEPENDS_ON -> ConnectionConfig @router.delete("/{connection_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_connection(connection_id: str, db: Session = Depends(get_db)): with belief_scope("ConnectionsRouter.delete_connection", f"id={connection_id}"): _ensure_connections_schema(db) - db_connection = db.query(ConnectionConfig).filter(ConnectionConfig.id == connection_id).first() + db_connection = ( + db.query(ConnectionConfig) + .filter(ConnectionConfig.id == connection_id) + .first() + ) if not db_connection: - logger.error(f"[ConnectionsRouter.delete_connection][State] Connection {connection_id} not found") + logger.error( + f"[ConnectionsRouter.delete_connection][State] Connection {connection_id} not found" + ) raise HTTPException(status_code=404, detail="Connection not found") db.delete(db_connection) db.commit() - logger.info(f"[ConnectionsRouter.delete_connection][Success] Deleted connection {connection_id}") + logger.info( + f"[ConnectionsRouter.delete_connection][Success] Deleted connection {connection_id}" + ) return + + # [/DEF:delete_connection:Function] # [/DEF:ConnectionsRouter:Module] diff --git a/backend/src/api/routes/dataset_review.py b/backend/src/api/routes/dataset_review.py index 29752d42..90f52fbf 100644 --- a/backend/src/api/routes/dataset_review.py +++ b/backend/src/api/routes/dataset_review.py @@ -25,7 +25,12 @@ from sqlalchemy.orm import Session from src.core.database import get_db from src.core.logger import belief_scope, logger -from src.dependencies import get_config_manager, get_current_user, get_task_manager, has_permission +from src.dependencies import ( + get_config_manager, + get_current_user, + get_task_manager, + has_permission, +) from src.models.auth import User from src.models.dataset_review import ( AnswerKind, @@ -84,6 +89,8 @@ class StartSessionRequest(BaseModel): source_kind: str = Field(..., pattern="^(superset_link|dataset_selection)$") source_input: str = Field(..., min_length=1) environment_id: str = Field(..., min_length=1) + + # [/DEF:StartSessionRequest:Class] @@ -93,6 +100,8 @@ class StartSessionRequest(BaseModel): class UpdateSessionRequest(BaseModel): status: SessionStatus note: Optional[str] = None + + # [/DEF:UpdateSessionRequest:Class] @@ -105,6 +114,8 @@ class SessionCollectionResponse(BaseModel): page: int page_size: int has_next: bool + + # [/DEF:SessionCollectionResponse:Class] @@ -120,6 +131,8 @@ class ExportArtifactResponse(BaseModel): created_by_user_id: str created_at: Optional[str] = None content: Dict[str, Any] + + # [/DEF:ExportArtifactResponse:Class] @@ -133,6 +146,8 @@ class FieldSemanticUpdateRequest(BaseModel): display_format: Optional[str] = None lock_field: bool = False resolution_note: Optional[str] = None + + # [/DEF:FieldSemanticUpdateRequest:Class] @@ -141,6 +156,8 @@ class FieldSemanticUpdateRequest(BaseModel): # @PURPOSE: Request DTO for thumbs up/down feedback persistence on AI-assisted content. class FeedbackRequest(BaseModel): feedback: str = Field(..., pattern="^(up|down)$") + + # [/DEF:FeedbackRequest:Class] @@ -151,6 +168,8 @@ class ClarificationAnswerRequest(BaseModel): question_id: str = Field(..., min_length=1) answer_kind: AnswerKind answer_value: Optional[str] = None + + # [/DEF:ClarificationAnswerRequest:Class] @@ -165,6 +184,8 @@ class ClarificationSessionSummaryResponse(BaseModel): resolved_count: int remaining_count: int summary_delta: Optional[str] = None + + # [/DEF:ClarificationSessionSummaryResponse:Class] @@ -174,6 +195,8 @@ class ClarificationSessionSummaryResponse(BaseModel): class ClarificationStateResponse(BaseModel): clarification_session: ClarificationSessionSummaryResponse current_question: Optional[ClarificationQuestionDto] = None + + # [/DEF:ClarificationStateResponse:Class] @@ -184,6 +207,8 @@ class ClarificationAnswerResultResponse(BaseModel): clarification_state: ClarificationStateResponse session: SessionSummary changed_findings: List[ValidationFindingDto] + + # [/DEF:ClarificationAnswerResultResponse:Class] @@ -193,6 +218,8 @@ class ClarificationAnswerResultResponse(BaseModel): class FeedbackResponse(BaseModel): target_id: str feedback: str + + # [/DEF:FeedbackResponse:Class] @@ -201,6 +228,8 @@ class FeedbackResponse(BaseModel): # @PURPOSE: Optional request DTO for explicit mapping approval audit notes. class ApproveMappingRequest(BaseModel): approval_note: Optional[str] = None + + # [/DEF:ApproveMappingRequest:Class] @@ -211,6 +240,8 @@ class BatchApproveSemanticItemRequest(BaseModel): field_id: str = Field(..., min_length=1) candidate_id: str = Field(..., min_length=1) lock_field: bool = False + + # [/DEF:BatchApproveSemanticItemRequest:Class] @@ -219,6 +250,8 @@ class BatchApproveSemanticItemRequest(BaseModel): # @PURPOSE: Request DTO for explicit batch semantic approvals inside one owned session scope. class BatchApproveSemanticRequest(BaseModel): items: List[BatchApproveSemanticItemRequest] = Field(..., min_length=1) + + # [/DEF:BatchApproveSemanticRequest:Class] @@ -228,6 +261,8 @@ class BatchApproveSemanticRequest(BaseModel): class BatchApproveMappingRequest(BaseModel): mapping_ids: List[str] = Field(..., min_length=1) approval_note: Optional[str] = None + + # [/DEF:BatchApproveMappingRequest:Class] @@ -238,6 +273,8 @@ class PreviewEnqueueResultResponse(BaseModel): session_id: str preview_status: str task_id: Optional[str] = None + + # [/DEF:PreviewEnqueueResultResponse:Class] @@ -246,6 +283,8 @@ class PreviewEnqueueResultResponse(BaseModel): # @PURPOSE: Contract-compliant wrapper for execution mapping list responses. class MappingCollectionResponse(BaseModel): items: List[ExecutionMappingDto] + + # [/DEF:MappingCollectionResponse:Class] @@ -254,8 +293,13 @@ class MappingCollectionResponse(BaseModel): # @PURPOSE: Request DTO for one manual execution-mapping override update without introducing unrelated bulk mutation semantics. class UpdateExecutionMappingRequest(BaseModel): effective_value: Optional[Any] = None - mapping_method: Optional[str] = Field(default=None, pattern="^(manual_override|direct_match|heuristic_match|semantic_match)$") + mapping_method: Optional[str] = Field( + default=None, + pattern="^(manual_override|direct_match|heuristic_match|semantic_match)$", + ) transformation_note: Optional[str] = None + + # [/DEF:UpdateExecutionMappingRequest:Class] @@ -265,6 +309,8 @@ class UpdateExecutionMappingRequest(BaseModel): class LaunchDatasetResponse(BaseModel): run_context: DatasetRunContextDto redirect_url: str + + # [/DEF:LaunchDatasetResponse:Class] @@ -280,6 +326,8 @@ def _require_auto_review_flag(config_manager=Depends(get_config_manager)) -> boo detail="Dataset auto review feature is disabled", ) return True + + # [/DEF:_require_auto_review_flag:Function] @@ -295,6 +343,8 @@ def _require_clarification_flag(config_manager=Depends(get_config_manager)) -> b detail="Dataset clarification feature is disabled", ) return True + + # [/DEF:_require_clarification_flag:Function] @@ -310,6 +360,8 @@ def _require_execution_flag(config_manager=Depends(get_config_manager)) -> bool: detail="Dataset execution feature is disabled", ) return True + + # [/DEF:_require_execution_flag:Function] @@ -318,6 +370,8 @@ def _require_execution_flag(config_manager=Depends(get_config_manager)) -> bool: # @PURPOSE: Build repository dependency for dataset review session aggregate access. def _get_repository(db: Session = Depends(get_db)) -> DatasetReviewSessionRepository: return DatasetReviewSessionRepository(db) + + # [/DEF:_get_repository:Function] @@ -335,6 +389,8 @@ def _get_orchestrator( config_manager=config_manager, task_manager=task_manager, ) + + # [/DEF:_get_orchestrator:Function] @@ -346,15 +402,20 @@ def _get_clarification_engine( repository: DatasetReviewSessionRepository = Depends(_get_repository), ) -> ClarificationEngine: return ClarificationEngine(repository=repository) + + # [/DEF:_get_clarification_engine:Function] # [DEF:_serialize_session_summary:Function] # @COMPLEXITY: 2 # @PURPOSE: Map SQLAlchemy session aggregate root into stable API summary DTO. +# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] # @RELATION: [DEPENDS_ON] ->[SessionSummary] def _serialize_session_summary(session: DatasetReviewSession) -> SessionSummary: return SessionSummary.model_validate(session, from_attributes=True) + + # [/DEF:_serialize_session_summary:Function] @@ -364,6 +425,8 @@ def _serialize_session_summary(session: DatasetReviewSession) -> SessionSummary: # @RELATION: [DEPENDS_ON] ->[SessionDetail] def _serialize_session_detail(session: DatasetReviewSession) -> SessionDetail: return SessionDetail.model_validate(session, from_attributes=True) + + # [/DEF:_serialize_session_detail:Function] @@ -373,6 +436,8 @@ def _serialize_session_detail(session: DatasetReviewSession) -> SessionDetail: # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntryDto] def _serialize_semantic_field(field: SemanticFieldEntry) -> SemanticFieldEntryDto: return SemanticFieldEntryDto.model_validate(field, from_attributes=True) + + # [/DEF:_serialize_semantic_field:Function] @@ -401,6 +466,8 @@ def _serialize_clarification_question_payload( "updated_at": datetime.utcnow(), } ) + + # [/DEF:_serialize_clarification_question_payload:Function] @@ -421,8 +488,12 @@ def _serialize_clarification_state( remaining_count=state.clarification_session.remaining_count, summary_delta=state.clarification_session.summary_delta, ), - current_question=_serialize_clarification_question_payload(state.current_question), + current_question=_serialize_clarification_question_payload( + state.current_question + ), ) + + # [/DEF:_serialize_clarification_state:Function] @@ -446,8 +517,12 @@ def _get_owned_session_or_404( "Dataset review session not found in current ownership scope", extra={"session_id": session_id, "user_id": current_user.id}, ) - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Session not found" + ) return session + + # [/DEF:_get_owned_session_or_404:Function] @@ -469,6 +544,8 @@ def _require_owner_mutation_scope( detail="Only the owner can mutate dataset review state", ) return session + + # [/DEF:_require_owner_mutation_scope:Function] @@ -492,6 +569,8 @@ def _record_session_event( event_summary=event_summary, event_details=event_details or {}, ) + + # [/DEF:_record_session_event:Function] @@ -510,7 +589,11 @@ def _get_owned_mapping_or_404( for mapping in session.execution_mappings: if mapping.mapping_id == mapping_id: return mapping - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Execution mapping not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Execution mapping not found" + ) + + # [/DEF:_get_owned_mapping_or_404:Function] @@ -529,7 +612,11 @@ def _get_owned_field_or_404( for field in session.semantic_fields: if field.field_id == field_id: return field - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Semantic field not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Semantic field not found" + ) + + # [/DEF:_get_owned_field_or_404:Function] @@ -541,12 +628,17 @@ def _get_latest_clarification_session_or_404( session: DatasetReviewSession, ) -> ClarificationSession: if not session.clarification_sessions: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Clarification session not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Clarification session not found", + ) return sorted( session.clarification_sessions, key=lambda item: (item.started_at, item.clarification_session_id), reverse=True, )[0] + + # [/DEF:_get_latest_clarification_session_or_404:Function] @@ -561,6 +653,8 @@ def _map_candidate_provenance(candidate: SemanticCandidate) -> FieldProvenance: if str(candidate.match_type.value) == "generated": return FieldProvenance.AI_GENERATED return FieldProvenance.FUZZY_INFERRED + + # [/DEF:_map_candidate_provenance:Function] @@ -569,7 +663,9 @@ def _map_candidate_provenance(candidate: SemanticCandidate) -> FieldProvenance: # @PURPOSE: Resolve the semantic source version for one accepted candidate from the loaded session aggregate. # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] # @RELATION: [DEPENDS_ON] ->[SemanticSource] -def _resolve_candidate_source_version(field: SemanticFieldEntry, source_id: Optional[str]) -> Optional[str]: +def _resolve_candidate_source_version( + field: SemanticFieldEntry, source_id: Optional[str] +) -> Optional[str]: if not source_id: return None session = getattr(field, "session", None) @@ -579,6 +675,8 @@ def _resolve_candidate_source_version(field: SemanticFieldEntry, source_id: Opti if source.source_id == source_id: return source.source_version return None + + # [/DEF:_resolve_candidate_source_version:Function] @@ -603,11 +701,18 @@ def _update_semantic_field_state( selected_candidate = None if request.candidate_id: selected_candidate = next( - (candidate for candidate in field.candidates if candidate.candidate_id == request.candidate_id), + ( + candidate + for candidate in field.candidates + if candidate.candidate_id == request.candidate_id + ), None, ) if selected_candidate is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Semantic candidate not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Semantic candidate not found", + ) if has_manual_override: field.verbose_name = request.verbose_name @@ -631,7 +736,9 @@ def _update_semantic_field_state( field.display_format = selected_candidate.proposed_display_format field.provenance = _map_candidate_provenance(selected_candidate) field.source_id = selected_candidate.source_id - field.source_version = _resolve_candidate_source_version(field, selected_candidate.source_id) + field.source_version = _resolve_candidate_source_version( + field, selected_candidate.source_id + ) field.confidence_rank = selected_candidate.candidate_rank field.is_locked = bool(request.lock_field or field.is_locked) field.has_conflict = len(field.candidates) > 1 @@ -649,6 +756,8 @@ def _update_semantic_field_state( status_code=status.HTTP_400_BAD_REQUEST, detail="Provide candidate_id or at least one manual override field", ) + + # [/DEF:_update_semantic_field_state:Function] @@ -658,6 +767,8 @@ def _update_semantic_field_state( # @RELATION: [DEPENDS_ON] ->[ExecutionMappingDto] def _serialize_execution_mapping(mapping: ExecutionMapping) -> ExecutionMappingDto: return ExecutionMappingDto.model_validate(mapping, from_attributes=True) + + # [/DEF:_serialize_execution_mapping:Function] @@ -667,6 +778,8 @@ def _serialize_execution_mapping(mapping: ExecutionMapping) -> ExecutionMappingD # @RELATION: [DEPENDS_ON] ->[DatasetRunContextDto] def _serialize_run_context(run_context) -> DatasetRunContextDto: return DatasetRunContextDto.model_validate(run_context, from_attributes=True) + + # [/DEF:_serialize_run_context:Function] @@ -688,6 +801,8 @@ def _build_sql_lab_redirect_url(environment_url: str, sql_lab_session_ref: str) detail="SQL Lab session reference is missing", ) return f"{base_url}/superset/sqllab?queryId={session_ref}" + + # [/DEF:_build_sql_lab_redirect_url:Function] @@ -695,9 +810,13 @@ def _build_sql_lab_redirect_url(environment_url: str, sql_lab_session_ref: str) # @COMPLEXITY: 2 # @PURPOSE: Produce session documentation export content from current persisted review state. # @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] -def _build_documentation_export(session: DatasetReviewSession, export_format: ArtifactFormat) -> Dict[str, Any]: +def _build_documentation_export( + session: DatasetReviewSession, export_format: ArtifactFormat +) -> Dict[str, Any]: profile = session.profile - findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code)) + findings = sorted( + session.findings, key=lambda item: (item.severity.value, item.code) + ) if export_format == ArtifactFormat.MARKDOWN: lines = [ f"# Dataset Review: {session.dataset_ref}", @@ -724,7 +843,8 @@ def _build_documentation_export(session: DatasetReviewSession, export_format: Ar else: content = { "session": _serialize_session_summary(session).model_dump(mode="json"), - "profile": profile and { + "profile": profile + and { "dataset_name": profile.dataset_name, "business_summary": profile.business_summary, "confidence_state": profile.confidence_state.value, @@ -743,6 +863,8 @@ def _build_documentation_export(session: DatasetReviewSession, export_format: Ar } storage_ref = f"inline://dataset-review/{session.session_id}/documentation.json" return {"storage_ref": storage_ref, "content": content} + + # [/DEF:_build_documentation_export:Function] @@ -750,8 +872,12 @@ def _build_documentation_export(session: DatasetReviewSession, export_format: Ar # @COMPLEXITY: 2 # @PURPOSE: Produce validation-focused export content from persisted findings and readiness state. # @RELATION: [DEPENDS_ON] ->[DatasetReviewSession] -def _build_validation_export(session: DatasetReviewSession, export_format: ArtifactFormat) -> Dict[str, Any]: - findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code)) +def _build_validation_export( + session: DatasetReviewSession, export_format: ArtifactFormat +) -> Dict[str, Any]: + findings = sorted( + session.findings, key=lambda item: (item.severity.value, item.code) + ) if export_format == ArtifactFormat.MARKDOWN: lines = [ f"# Validation Report: {session.dataset_ref}", @@ -790,6 +916,8 @@ def _build_validation_export(session: DatasetReviewSession, export_format: Artif } storage_ref = f"inline://dataset-review/{session.session_id}/validation.json" return {"storage_ref": storage_ref, "content": content} + + # [/DEF:_build_validation_export:Function] @@ -823,13 +951,15 @@ async def list_sessions( page_size=page_size, has_next=end < len(sessions), ) + + # [/DEF:list_sessions:Function] # [DEF:start_session:Function] # @COMPLEXITY: 4 # @PURPOSE: Start a new dataset review session from a Superset link or dataset selection. -# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.start_session] +# @RELATION: [CALLS] ->[start_session:Function] # @PRE: feature flag enabled, user authenticated, and request body valid. # @POST: returns persisted session summary scoped to the authenticated user. # @SIDE_EFFECT: persists session/profile/findings and may enqueue recovery task. @@ -864,10 +994,16 @@ async def start_session( extra={"user_id": current_user.id, "error": str(exc)}, ) detail = str(exc) - status_code = status.HTTP_404_NOT_FOUND if detail == "Environment not found" else status.HTTP_400_BAD_REQUEST + status_code = ( + status.HTTP_404_NOT_FOUND + if detail == "Environment not found" + else status.HTTP_400_BAD_REQUEST + ) raise HTTPException(status_code=status_code, detail=detail) from exc return _serialize_session_summary(result.session) + + # [/DEF:start_session:Function] @@ -891,6 +1027,8 @@ async def get_session_detail( with belief_scope("dataset_review.get_session_detail"): session = _get_owned_session_or_404(repository, session_id, current_user) return _serialize_session_detail(session) + + # [/DEF:get_session_detail:Function] @@ -923,7 +1061,11 @@ async def update_session( session.status = request.status if request.status == SessionStatus.PAUSED: session.recommended_action = RecommendedAction.RESUME_SESSION - elif request.status in {SessionStatus.ARCHIVED, SessionStatus.CANCELLED, SessionStatus.COMPLETED}: + elif request.status in { + SessionStatus.ARCHIVED, + SessionStatus.CANCELLED, + SessionStatus.COMPLETED, + }: session.active_task_id = None repository.db.commit() @@ -937,6 +1079,8 @@ async def update_session( event_details={"status": session.status.value}, ) return _serialize_session_summary(session) + + # [/DEF:update_session:Function] @@ -992,6 +1136,8 @@ async def delete_session( event_details={"hard_delete": False}, ) return Response(status_code=status.HTTP_204_NO_CONTENT) + + # [/DEF:delete_session:Function] @@ -1019,7 +1165,10 @@ async def export_documentation( ): with belief_scope("dataset_review.export_documentation"): if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Only json and markdown exports are supported") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Only json and markdown exports are supported", + ) session = _get_owned_session_or_404(repository, session_id, current_user) export_payload = _build_documentation_export(session, format) return ExportArtifactResponse( @@ -1031,6 +1180,8 @@ async def export_documentation( created_by_user_id=current_user.id, content=export_payload["content"], ) + + # [/DEF:export_documentation:Function] @@ -1058,7 +1209,10 @@ async def export_validation( ): with belief_scope("dataset_review.export_validation"): if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Only json and markdown exports are supported") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Only json and markdown exports are supported", + ) session = _get_owned_session_or_404(repository, session_id, current_user) export_payload = _build_validation_export(session, format) return ExportArtifactResponse( @@ -1070,13 +1224,15 @@ async def export_validation( created_by_user_id=current_user.id, content=export_payload["content"], ) + + # [/DEF:export_validation:Function] # [DEF:get_clarification_state:Function] # @COMPLEXITY: 4 # @PURPOSE: Return the current clarification session summary and one active question payload. -# @RELATION: [CALLS] ->[ClarificationEngine.build_question_payload] +# @RELATION: [CALLS] ->[build_question_payload:Function] # @PRE: Session is accessible to current user and clarification feature is enabled. # @POST: Returns at most one active clarification question with why_it_matters, current_guess, and ordered options. # @SIDE_EFFECT: May normalize clarification pointer and readiness state in persistence. @@ -1108,13 +1264,15 @@ async def get_clarification_state( changed_findings=[], ) ) + + # [/DEF:get_clarification_state:Function] # [DEF:resume_clarification:Function] # @COMPLEXITY: 4 # @PURPOSE: Resume clarification mode on the highest-priority unresolved question for an owned session. -# @RELATION: [CALLS] ->[ClarificationEngine.build_question_payload] +# @RELATION: [CALLS] ->[build_question_payload:Function] # @PRE: Session belongs to the current owner and clarification feature is enabled. # @POST: Clarification session enters active state with one current question or completes deterministically when no unresolved items remain. # @SIDE_EFFECT: Mutates clarification pointer, readiness, and recommended action. @@ -1147,13 +1305,15 @@ async def resume_clarification( changed_findings=[], ) ) + + # [/DEF:resume_clarification:Function] # [DEF:record_clarification_answer:Function] # @COMPLEXITY: 4 # @PURPOSE: Persist one clarification answer before advancing the active pointer or readiness state. -# @RELATION: [CALLS] ->[ClarificationEngine.record_answer] +# @RELATION: [CALLS] ->[record_answer:Function] # @PRE: Target question is the session's active clarification question and current user owns the session. # @POST: Answer is persisted, changed findings are returned, and unresolved skipped/expert-review questions remain visible. # @SIDE_EFFECT: Inserts answer row and mutates clarification/session state. @@ -1188,7 +1348,9 @@ async def record_clarification_answer( ) ) except ValueError as exc: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc) + ) from exc return ClarificationAnswerResultResponse( clarification_state=_serialize_clarification_state(result), @@ -1198,6 +1360,8 @@ async def record_clarification_answer( for item in result.changed_findings ], ) + + # [/DEF:record_clarification_answer:Function] @@ -1246,6 +1410,8 @@ async def update_field_semantic( }, ) return _serialize_semantic_field(field) + + # [/DEF:update_field_semantic:Function] @@ -1288,6 +1454,8 @@ async def lock_field_semantic( event_details={"field_id": field.field_id}, ) return _serialize_semantic_field(field) + + # [/DEF:lock_field_semantic:Function] @@ -1333,6 +1501,8 @@ async def unlock_field_semantic( event_details={"field_id": field.field_id}, ) return _serialize_semantic_field(field) + + # [/DEF:unlock_field_semantic:Function] @@ -1367,7 +1537,9 @@ async def approve_batch_semantic_fields( field = _get_owned_field_or_404(session, item.field_id) updated_field = _update_semantic_field_state( field, - FieldSemanticUpdateRequest(candidate_id=item.candidate_id, lock_field=item.lock_field), + FieldSemanticUpdateRequest( + candidate_id=item.candidate_id, lock_field=item.lock_field + ), changed_by="user", ) updated_fields.append(updated_field) @@ -1387,6 +1559,8 @@ async def approve_batch_semantic_fields( }, ) return [_serialize_semantic_field(field) for field in updated_fields] + + # [/DEF:approve_batch_semantic_fields:Function] @@ -1415,8 +1589,13 @@ async def list_execution_mappings( with belief_scope("dataset_review.list_execution_mappings"): session = _get_owned_session_or_404(repository, session_id, current_user) return MappingCollectionResponse( - items=[_serialize_execution_mapping(item) for item in session.execution_mappings] + items=[ + _serialize_execution_mapping(item) + for item in session.execution_mappings + ] ) + + # [/DEF:list_execution_mappings:Function] @@ -1456,7 +1635,9 @@ async def update_execution_mapping( ) mapping.effective_value = request.effective_value - mapping.mapping_method = MappingMethod(request.mapping_method or MappingMethod.MANUAL_OVERRIDE.value) + mapping.mapping_method = MappingMethod( + request.mapping_method or MappingMethod.MANUAL_OVERRIDE.value + ) mapping.transformation_note = request.transformation_note mapping.approval_state = ApprovalState.APPROVED mapping.approved_by_user_id = current_user.id @@ -1491,6 +1672,8 @@ async def update_execution_mapping( }, ) return _serialize_execution_mapping(mapping) + + # [/DEF:update_execution_mapping:Function] @@ -1544,6 +1727,8 @@ async def approve_execution_mapping( }, ) return _serialize_execution_mapping(mapping) + + # [/DEF:approve_execution_mapping:Function] @@ -1603,13 +1788,15 @@ async def approve_batch_execution_mappings( }, ) return [_serialize_execution_mapping(mapping) for mapping in updated_mappings] + + # [/DEF:approve_batch_execution_mappings:Function] # [DEF:trigger_preview_generation:Function] # @COMPLEXITY: 4 # @PURPOSE: Trigger Superset-side preview compilation for the current owned execution context. -# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.prepare_launch_preview] +# @RELATION: [CALLS] ->[prepare_launch_preview:Function] # @PRE: Session belongs to the current owner and required mapping inputs are available. # @POST: Returns the compiled preview directly for synchronous success or enqueue-state shape when preview generation remains pending. # @SIDE_EFFECT: Persists preview attempt and updates readiness state. @@ -1640,8 +1827,10 @@ async def trigger_preview_generation( except ValueError as exc: detail = str(exc) status_code = ( - status.HTTP_404_NOT_FOUND if detail in {"Session not found", "Environment not found"} - else status.HTTP_409_CONFLICT if detail.startswith("Preview blocked:") + status.HTTP_404_NOT_FOUND + if detail in {"Session not found", "Environment not found"} + else status.HTTP_409_CONFLICT + if detail.startswith("Preview blocked:") else status.HTTP_400_BAD_REQUEST ) raise HTTPException(status_code=status_code, detail=detail) from exc @@ -1656,13 +1845,15 @@ async def trigger_preview_generation( response.status_code = status.HTTP_200_OK return CompiledPreviewDto.model_validate(result.preview, from_attributes=True) + + # [/DEF:trigger_preview_generation:Function] # [DEF:launch_dataset:Function] # @COMPLEXITY: 4 # @PURPOSE: Execute the current owned session launch handoff through the orchestrator and return audited SQL Lab run context. -# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.launch_dataset] +# @RELATION: [CALLS] ->[launch_dataset:Function] # @PRE: Session belongs to the current owner, execution feature is enabled, and launch gates are satisfied or a deterministic conflict is returned. # @POST: Returns persisted run context plus redirect URL when launch handoff is accepted. # @SIDE_EFFECT: Persists launch audit snapshot and may trigger SQL Lab session creation. @@ -1703,7 +1894,9 @@ async def launch_dataset( raise HTTPException(status_code=status_code, detail=detail) from exc environment = config_manager.get_environment(result.session.environment_id) - environment_url = getattr(environment, "url", "") if environment is not None else "" + environment_url = ( + getattr(environment, "url", "") if environment is not None else "" + ) return LaunchDatasetResponse( run_context=_serialize_run_context(result.run_context), redirect_url=_build_sql_lab_redirect_url( @@ -1711,6 +1904,8 @@ async def launch_dataset( sql_lab_session_ref=result.run_context.sql_lab_session_ref, ), ) + + # [/DEF:launch_dataset:Function] @@ -1752,6 +1947,8 @@ async def record_field_feedback( event_details={"field_id": field.field_id, "feedback": request.feedback}, ) return FeedbackResponse(target_id=field.field_id, feedback=request.feedback) + + # [/DEF:record_field_feedback:Function] @@ -1784,13 +1981,23 @@ async def record_clarification_feedback( _require_owner_mutation_scope(session, current_user) clarification_session = _get_latest_clarification_session_or_404(session) question = next( - (item for item in clarification_session.questions if item.question_id == question_id), + ( + item + for item in clarification_session.questions + if item.question_id == question_id + ), None, ) if question is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Clarification question not found") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Clarification question not found", + ) if question.answer is None: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Clarification answer not found") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Clarification answer not found", + ) question.answer.user_feedback = request.feedback repository.db.commit() _record_session_event( @@ -1799,9 +2006,16 @@ async def record_clarification_feedback( current_user, event_type="clarification_feedback_recorded", event_summary="Clarification feedback persisted", - event_details={"question_id": question.question_id, "feedback": request.feedback}, + event_details={ + "question_id": question.question_id, + "feedback": request.feedback, + }, ) - return FeedbackResponse(target_id=question.question_id, feedback=request.feedback) + return FeedbackResponse( + target_id=question.question_id, feedback=request.feedback + ) + + # [/DEF:record_clarification_feedback:Function] -# [/DEF:DatasetReviewApi:Module] \ No newline at end of file +# [/DEF:DatasetReviewApi:Module] diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 31b7b305..4a6c8b20 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -31,21 +31,25 @@ def _is_valid_runtime_api_key(value: Optional[str]) -> bool: if key in {"********", "EMPTY_OR_NONE"}: return False return len(key) >= 16 + + # [/DEF:_is_valid_runtime_api_key:Function] + # [DEF:get_providers:Function] # @PURPOSE: Retrieve all LLM provider configurations. # @PRE: User is authenticated. # @POST: Returns list of LLMProviderConfig. @router.get("/providers", response_model=List[LLMProviderConfig]) async def get_providers( - current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db) + current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db) ): """ Get all LLM provider configurations. """ - logger.info(f"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}") + logger.info( + f"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}" + ) service = LLMProviderService(db) providers = service.get_all_providers() return [ @@ -56,9 +60,12 @@ async def get_providers( base_url=p.base_url, api_key="********", default_model=p.default_model, - is_active=p.is_active - ) for p in providers + is_active=p.is_active, + ) + for p in providers ] + + # [/DEF:get_providers:Function] @@ -68,8 +75,7 @@ async def get_providers( # @POST: configured=true only when an active provider with valid decrypted key exists. @router.get("/status") async def get_llm_status( - current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db) + current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db) ): service = LLMProviderService(db) providers = service.get_all_providers() @@ -90,17 +96,22 @@ async def get_llm_status( "provider_type": active_provider.provider_type, "default_model": active_provider.default_model, } + + # [/DEF:get_llm_status:Function] + # [DEF:create_provider:Function] # @PURPOSE: Create a new LLM provider configuration. # @PRE: User is authenticated and has admin permissions. # @POST: Returns the created LLMProviderConfig. -@router.post("/providers", response_model=LLMProviderConfig, status_code=status.HTTP_201_CREATED) +@router.post( + "/providers", response_model=LLMProviderConfig, status_code=status.HTTP_201_CREATED +) async def create_provider( config: LLMProviderConfig, current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db) + db: Session = Depends(get_db), ): """ Create a new LLM provider configuration. @@ -114,10 +125,13 @@ async def create_provider( base_url=provider.base_url, api_key="********", default_model=provider.default_model, - is_active=provider.is_active + is_active=provider.is_active, ) + + # [/DEF:create_provider:Function] + # [DEF:update_provider:Function] # @PURPOSE: Update an existing LLM provider configuration. # @PRE: User is authenticated and has admin permissions. @@ -127,7 +141,7 @@ async def update_provider( provider_id: str, config: LLMProviderConfig, current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db) + db: Session = Depends(get_db), ): """ Update an existing LLM provider configuration. @@ -136,7 +150,7 @@ async def update_provider( provider = service.update_provider(provider_id, config) if not provider: raise HTTPException(status_code=404, detail="Provider not found") - + return LLMProviderConfig( id=provider.id, provider_type=LLMProviderType(provider.provider_type), @@ -144,10 +158,13 @@ async def update_provider( base_url=provider.base_url, api_key="********", default_model=provider.default_model, - is_active=provider.is_active + is_active=provider.is_active, ) + + # [/DEF:update_provider:Function] + # [DEF:delete_provider:Function] # @PURPOSE: Delete an LLM provider configuration. # @PRE: User is authenticated and has admin permissions. @@ -156,7 +173,7 @@ async def update_provider( async def delete_provider( provider_id: str, current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db) + db: Session = Depends(get_db), ): """ Delete an LLM provider configuration. @@ -165,8 +182,11 @@ async def delete_provider( if not service.delete_provider(provider_id): raise HTTPException(status_code=404, detail="Provider not found") return + + # [/DEF:delete_provider:Function] + # [DEF:test_connection:Function] # @PURPOSE: Test connection to an LLM provider. # @PRE: User is authenticated. @@ -175,76 +195,87 @@ async def delete_provider( async def test_connection( provider_id: str, current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db) + db: Session = Depends(get_db), ): - logger.info(f"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}") + logger.info( + f"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}" + ) """ Test connection to an LLM provider. """ from ...plugins.llm_analysis.service import LLMClient + service = LLMProviderService(db) db_provider = service.get_provider(provider_id) if not db_provider: raise HTTPException(status_code=404, detail="Provider not found") - + api_key = service.get_decrypted_api_key(provider_id) - + # Check if API key was successfully decrypted if not api_key: - logger.error(f"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}") + logger.error( + f"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}" + ) raise HTTPException( status_code=500, - detail="Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key." + detail="Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.", ) - + client = LLMClient( provider_type=LLMProviderType(db_provider.provider_type), api_key=api_key, base_url=db_provider.base_url, - default_model=db_provider.default_model + default_model=db_provider.default_model, ) - + try: await client.test_runtime_connection() return {"success": True, "message": "Connection successful"} except Exception as e: return {"success": False, "error": str(e)} + + # [/DEF:test_connection:Function] + # [DEF:test_provider_config:Function] # @PURPOSE: Test connection with a provided configuration (not yet saved). # @PRE: User is authenticated. # @POST: Returns success status and message. @router.post("/providers/test") async def test_provider_config( - config: LLMProviderConfig, - current_user: User = Depends(get_current_active_user) + config: LLMProviderConfig, current_user: User = Depends(get_current_active_user) ): """ Test connection with a provided configuration. """ from ...plugins.llm_analysis.service import LLMClient - logger.info(f"[llm_routes][test_provider_config][Action] Testing config for {config.name}") - + + logger.info( + f"[llm_routes][test_provider_config][Action] Testing config for {config.name}" + ) + # Check if API key is provided if not config.api_key or config.api_key == "********": raise HTTPException( - status_code=400, - detail="API key is required for testing connection" + status_code=400, detail="API key is required for testing connection" ) - + client = LLMClient( provider_type=config.provider_type, api_key=config.api_key, base_url=config.base_url, - default_model=config.default_model + default_model=config.default_model, ) - + try: await client.test_runtime_connection() return {"success": True, "message": "Connection successful"} except Exception as e: return {"success": False, "error": str(e)} + + # [/DEF:test_provider_config:Function] -# [/DEF:backend/src/api/routes/llm.py] +# [/DEF:backend/src/api/routes/llm.py:Module] diff --git a/backend/src/api/routes/reports.py b/backend/src/api/routes/reports.py index 45a73c8d..05a79323 100644 --- a/backend/src/api/routes/reports.py +++ b/backend/src/api/routes/reports.py @@ -3,8 +3,10 @@ # @SEMANTICS: api, reports, list, detail, pagination, filters # @PURPOSE: FastAPI router for unified task report list and detail retrieval endpoints. # @LAYER: UI (API) -# @RELATION: DEPENDS_ON -> [backend.src.services.reports.report_service.ReportsService] -# @RELATION: DEPENDS_ON -> [AppDependencies] +# @RELATION: [DEPENDS_ON] ->[ReportsService:Class] +# @RELATION: [DEPENDS_ON] ->[get_task_manager:Function] +# @RELATION: [DEPENDS_ON] ->[get_clean_release_repository:Function] +# @RELATION: [DEPENDS_ON] ->[has_permission:Function] # @INVARIANT: Endpoints are read-only and do not trigger long-running tasks. # @PRE: Reports service and dependencies are initialized. # @POST: Router is configured and endpoints are ready for registration. @@ -17,10 +19,20 @@ from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Query, status -from ...dependencies import get_task_manager, has_permission, get_clean_release_repository +from ...dependencies import ( + get_task_manager, + has_permission, + get_clean_release_repository, +) from ...core.task_manager import TaskManager from ...core.logger import belief_scope -from ...models.report import ReportCollection, ReportDetailView, ReportQuery, ReportStatus, TaskType +from ...models.report import ( + ReportCollection, + ReportDetailView, + ReportQuery, + ReportStatus, + TaskType, +) from ...services.clean_release.repository import CleanReleaseRepository from ...services.reports.report_service import ReportsService # [/SECTION] @@ -60,6 +72,8 @@ def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List: }, ) return parsed + + # [/DEF:_parse_csv_enum_list:Function] @@ -69,6 +83,9 @@ def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List: # @PRE: authenticated/authorized request and validated query params. # @POST: returns {items,total,page,page_size,has_next,applied_filters}. # @POST: deterministic error payload for invalid filters. +# @RELATION: [CALLS] ->[_parse_csv_enum_list:Function] +# @RELATION: [DEPENDS_ON] ->[ReportQuery:Class] +# @RELATION: [DEPENDS_ON] ->[ReportsService:Class] # # @TEST_CONTRACT: ListReportsApi -> # { @@ -95,7 +112,9 @@ async def list_reports( sort_by: str = Query("updated_at"), sort_order: str = Query("desc"), task_manager: TaskManager = Depends(get_task_manager), - clean_release_repository: CleanReleaseRepository = Depends(get_clean_release_repository), + clean_release_repository: CleanReleaseRepository = Depends( + get_clean_release_repository + ), _=Depends(has_permission("tasks", "READ")), ): with belief_scope("list_reports"): @@ -125,8 +144,12 @@ async def list_reports( }, ) - service = ReportsService(task_manager, clean_release_repository=clean_release_repository) + service = ReportsService( + task_manager, clean_release_repository=clean_release_repository + ) return service.list_reports(query) + + # [/DEF:list_reports:Function] @@ -139,11 +162,15 @@ async def list_reports( async def get_report_detail( report_id: str, task_manager: TaskManager = Depends(get_task_manager), - clean_release_repository: CleanReleaseRepository = Depends(get_clean_release_repository), + clean_release_repository: CleanReleaseRepository = Depends( + get_clean_release_repository + ), _=Depends(has_permission("tasks", "READ")), ): with belief_scope("get_report_detail", f"report_id={report_id}"): - service = ReportsService(task_manager, clean_release_repository=clean_release_repository) + service = ReportsService( + task_manager, clean_release_repository=clean_release_repository + ) detail = service.get_report_detail(report_id) if not detail: raise HTTPException( @@ -151,6 +178,8 @@ async def get_report_detail( detail={"message": "Report not found", "code": "REPORT_NOT_FOUND"}, ) return detail + + # [/DEF:get_report_detail:Function] -# [/DEF:ReportsRouter:Module] \ No newline at end of file +# [/DEF:ReportsRouter:Module] diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index de1545c4..bf16bce8 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -4,8 +4,9 @@ # @SEMANTICS: settings, api, router, fastapi # @PURPOSE: Provides API endpoints for managing application settings and Superset environments. # @LAYER: UI (API) -# @RELATION: DEPENDS_ON -> [backend.src.core.config_manager.ConfigManager] -# @RELATION: DEPENDS_ON -> [backend.src.core.config_models] +# @RELATION: [DEPENDS_ON] ->[ConfigManager] +# @RELATION: [DEPENDS_ON] ->[get_config_manager:Function] +# @RELATION: [DEPENDS_ON] ->[has_permission:Function] # # @INVARIANT: All settings changes must be persisted via ConfigManager. # @PUBLIC_API: router @@ -413,6 +414,12 @@ class ConsolidatedSettingsResponse(BaseModel): # @PRE: Config manager is available. # @POST: Returns all consolidated settings. # @RETURN: ConsolidatedSettingsResponse - All settings categories. +# @RELATION: [DEPENDS_ON] ->[ConfigManager] +# @RELATION: [DEPENDS_ON] ->[LLMProviderService] +# @RELATION: [DEPENDS_ON] ->[AppConfigRecord] +# @RELATION: [DEPENDS_ON] ->[SessionLocal] +# @RELATION: [DEPENDS_ON] ->[has_permission:Function] +# @RELATION: [DEPENDS_ON] ->[normalize_llm_settings:Function] @router.get("/consolidated", response_model=ConsolidatedSettingsResponse) async def get_consolidated_settings( config_manager: ConfigManager = Depends(get_config_manager), diff --git a/backend/src/core/__tests__/test_config_manager_compat.py b/backend/src/core/__tests__/test_config_manager_compat.py index 45c0299f..74f48692 100644 --- a/backend/src/core/__tests__/test_config_manager_compat.py +++ b/backend/src/core/__tests__/test_config_manager_compat.py @@ -23,6 +23,8 @@ def test_get_payload_preserves_legacy_sections(): assert payload["settings"]["migration_sync_cron"] == "0 2 * * *" assert payload["notifications"]["smtp"]["host"] == "mail.local" + + # [/DEF:test_get_payload_preserves_legacy_sections:Function] @@ -71,10 +73,22 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows(): credentials_id="legacy-user", ) + # [DEF:_FakeQuery:Class] + # @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal query stub returning hardcoded existing environment record list for sync tests. + # @INVARIANT: all() always returns [existing_record]; no parameterization or filtering. class _FakeQuery: def all(self): return [existing_record] + # [/DEF:_FakeQuery:Class] + + # [DEF:_FakeSession:Class] + # @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal SQLAlchemy session stub that captures add/delete calls for environment sync assertions. + # @INVARIANT: query() always returns _FakeQuery; no real DB interaction. class _FakeSession: def query(self, model): return _FakeQuery() @@ -85,6 +99,8 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows(): def delete(self, value): deleted_records.append(value) + # [/DEF:_FakeSession:Class] + session = _FakeSession() config = AppConfig( environments=[ @@ -107,6 +123,8 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows(): assert added_records[0].url == "http://superset.local" assert added_records[0].credentials_id == "demo" assert deleted_records == [existing_record] + + # [/DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function] @@ -123,6 +141,11 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa closed = {"value": False} committed = {"value": False} + # [DEF:_FakeSession:Class] + # @RELATION: BINDS_TO -> [test_load_config_syncs_environment_records_from_existing_db_payload] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal session stub tracking commit/close signals for config load lifecycle assertions. + # @INVARIANT: No query or add semantics; only lifecycle signal tracking. class _FakeSession: def commit(self): committed["value"] = True @@ -130,6 +153,8 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa def close(self): closed["value"] = True + # [/DEF:_FakeSession:Class] + fake_session = _FakeSession() fake_record = SimpleNamespace( id="global", @@ -163,6 +188,8 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa assert sync_calls[0][1].environments[0].id == "dev" assert committed["value"] is True assert closed["value"] is True + + # [/DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function] # [/DEF:TestConfigManagerCompat:Module] diff --git a/backend/src/core/__tests__/test_throttled_scheduler.py b/backend/src/core/__tests__/test_throttled_scheduler.py index 8a8e7142..ef47ddfb 100644 --- a/backend/src/core/__tests__/test_throttled_scheduler.py +++ b/backend/src/core/__tests__/test_throttled_scheduler.py @@ -7,8 +7,10 @@ from src.core.scheduler import ThrottledSchedulerConfigurator # @COMPLEXITY: 3 # @PURPOSE: Unit tests for ThrottledSchedulerConfigurator distribution logic. + # [DEF:test_calculate_schedule_even_distribution:Function] # @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate even spacing across a two-hour scheduling window for three tasks. def test_calculate_schedule_even_distribution(): """ @TEST_SCENARIO: 3 tasks in a 2-hour window should be spaced 1 hour apart. @@ -17,18 +19,23 @@ def test_calculate_schedule_even_distribution(): end = time(3, 0) dashboards = ["d1", "d2", "d3"] today = date(2024, 1, 1) - - schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today) - + + schedule = ThrottledSchedulerConfigurator.calculate_schedule( + start, end, dashboards, today + ) + assert len(schedule) == 3 assert schedule[0] == datetime(2024, 1, 1, 1, 0) assert schedule[1] == datetime(2024, 1, 1, 2, 0) assert schedule[2] == datetime(2024, 1, 1, 3, 0) + # [/DEF:test_calculate_schedule_even_distribution:Function] + # [DEF:test_calculate_schedule_midnight_crossing:Function] # @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate scheduler correctly rolls timestamps into the next day across midnight. def test_calculate_schedule_midnight_crossing(): """ @TEST_SCENARIO: Window from 23:00 to 01:00 (next day). @@ -37,18 +44,23 @@ def test_calculate_schedule_midnight_crossing(): end = time(1, 0) dashboards = ["d1", "d2", "d3"] today = date(2024, 1, 1) - - schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today) - + + schedule = ThrottledSchedulerConfigurator.calculate_schedule( + start, end, dashboards, today + ) + assert len(schedule) == 3 assert schedule[0] == datetime(2024, 1, 1, 23, 0) assert schedule[1] == datetime(2024, 1, 2, 0, 0) assert schedule[2] == datetime(2024, 1, 2, 1, 0) + # [/DEF:test_calculate_schedule_midnight_crossing:Function] + # [DEF:test_calculate_schedule_single_task:Function] # @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate single-task schedule returns only the window start timestamp. def test_calculate_schedule_single_task(): """ @TEST_SCENARIO: Single task should be scheduled at start time. @@ -57,16 +69,21 @@ def test_calculate_schedule_single_task(): end = time(2, 0) dashboards = ["d1"] today = date(2024, 1, 1) - - schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today) - + + schedule = ThrottledSchedulerConfigurator.calculate_schedule( + start, end, dashboards, today + ) + assert len(schedule) == 1 assert schedule[0] == datetime(2024, 1, 1, 1, 0) + # [/DEF:test_calculate_schedule_single_task:Function] + # [DEF:test_calculate_schedule_empty_list:Function] # @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate empty dashboard list produces an empty schedule. def test_calculate_schedule_empty_list(): """ @TEST_SCENARIO: Empty dashboard list returns empty schedule. @@ -75,15 +92,20 @@ def test_calculate_schedule_empty_list(): end = time(2, 0) dashboards = [] today = date(2024, 1, 1) - - schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today) - + + schedule = ThrottledSchedulerConfigurator.calculate_schedule( + start, end, dashboards, today + ) + assert schedule == [] + # [/DEF:test_calculate_schedule_empty_list:Function] + # [DEF:test_calculate_schedule_zero_window:Function] # @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate zero-length window schedules all tasks at identical start timestamp. def test_calculate_schedule_zero_window(): """ @TEST_SCENARIO: Window start == end. All tasks at start time. @@ -92,31 +114,40 @@ def test_calculate_schedule_zero_window(): end = time(1, 0) dashboards = ["d1", "d2"] today = date(2024, 1, 1) - - schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today) - + + schedule = ThrottledSchedulerConfigurator.calculate_schedule( + start, end, dashboards, today + ) + assert len(schedule) == 2 assert schedule[0] == datetime(2024, 1, 1, 1, 0) assert schedule[1] == datetime(2024, 1, 1, 1, 0) + # [/DEF:test_calculate_schedule_zero_window:Function] + # [DEF:test_calculate_schedule_very_small_window:Function] # @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate sub-second interpolation when task count exceeds near-zero window granularity. def test_calculate_schedule_very_small_window(): """ @TEST_SCENARIO: Window smaller than number of tasks (in seconds). """ start = time(1, 0, 0) - end = time(1, 0, 1) # 1 second window + end = time(1, 0, 1) # 1 second window dashboards = ["d1", "d2", "d3"] today = date(2024, 1, 1) - - schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today) - + + schedule = ThrottledSchedulerConfigurator.calculate_schedule( + start, end, dashboards, today + ) + assert len(schedule) == 3 assert schedule[0] == datetime(2024, 1, 1, 1, 0, 0) - assert schedule[1] == datetime(2024, 1, 1, 1, 0, 0, 500000) # 0.5s + assert schedule[1] == datetime(2024, 1, 1, 1, 0, 0, 500000) # 0.5s assert schedule[2] == datetime(2024, 1, 1, 1, 0, 1) -# [/DEF:test_throttled_scheduler:Module]# [/DEF:test_calculate_schedule_very_small_window:Function] + +# [/DEF:test_calculate_schedule_very_small_window:Function] +# [/DEF:test_throttled_scheduler:Module] diff --git a/backend/src/core/auth/config.py b/backend/src/core/auth/config.py index 79cd193a..05f9d9de 100644 --- a/backend/src/core/auth/config.py +++ b/backend/src/core/auth/config.py @@ -2,6 +2,7 @@ # # @SEMANTICS: auth, config, settings, jwt, adfs # @PURPOSE: Centralized configuration for authentication and authorization. +# @COMPLEXITY: 2 # @LAYER: Core # @RELATION: DEPENDS_ON -> pydantic # diff --git a/backend/src/core/auth/jwt.py b/backend/src/core/auth/jwt.py index 9a8a3d74..fac215b5 100644 --- a/backend/src/core/auth/jwt.py +++ b/backend/src/core/auth/jwt.py @@ -4,8 +4,8 @@ # @SEMANTICS: jwt, token, session, auth # @PURPOSE: JWT token generation and validation logic. # @LAYER: Core -# @RELATION: DEPENDS_ON -> jose -# @RELATION: USES -> auth_config +# @RELATION: DEPENDS_ON -> [auth_config] +# @RELATION: USES -> [auth_config] # # @INVARIANT: Tokens must include expiration time and user identifier. @@ -17,11 +17,12 @@ from .config import auth_config from ..logger import belief_scope # [/SECTION] + # [DEF:create_access_token:Function] # @PURPOSE: Generates a new JWT access token. # @PRE: data dict contains 'sub' (user_id) and optional 'scopes' (roles). # @POST: Returns a signed JWT string. -# @RELATION: DEPENDS_ON -> auth_config +# @RELATION: DEPENDS_ON -> [auth_config] # # @PARAM: data (dict) - Payload data for the token. # @PARAM: expires_delta (Optional[timedelta]) - Custom expiration time. @@ -32,26 +33,37 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) - if expires_delta: expire = datetime.utcnow() + expires_delta else: - expire = datetime.utcnow() + timedelta(minutes=auth_config.ACCESS_TOKEN_EXPIRE_MINUTES) - + expire = datetime.utcnow() + timedelta( + minutes=auth_config.ACCESS_TOKEN_EXPIRE_MINUTES + ) + to_encode.update({"exp": expire}) - encoded_jwt = jwt.encode(to_encode, auth_config.SECRET_KEY, algorithm=auth_config.ALGORITHM) + encoded_jwt = jwt.encode( + to_encode, auth_config.SECRET_KEY, algorithm=auth_config.ALGORITHM + ) return encoded_jwt + + # [/DEF:create_access_token:Function] + # [DEF:decode_token:Function] # @PURPOSE: Decodes and validates a JWT token. # @PRE: token is a signed JWT string. # @POST: Returns the decoded payload if valid. -# @RELATION: DEPENDS_ON -> auth_config +# @RELATION: DEPENDS_ON -> [auth_config] # # @PARAM: token (str) - The JWT to decode. # @RETURN: dict - The decoded payload. # @THROW: jose.JWTError - If token is invalid or expired. def decode_token(token: str) -> dict: with belief_scope("decode_token"): - payload = jwt.decode(token, auth_config.SECRET_KEY, algorithms=[auth_config.ALGORITHM]) + payload = jwt.decode( + token, auth_config.SECRET_KEY, algorithms=[auth_config.ALGORITHM] + ) return payload + + # [/DEF:decode_token:Function] -# [/DEF:AuthJwtModule:Module] \ No newline at end of file +# [/DEF:AuthJwtModule:Module] diff --git a/backend/src/core/auth/oauth.py b/backend/src/core/auth/oauth.py index 1a462a58..04d6852e 100644 --- a/backend/src/core/auth/oauth.py +++ b/backend/src/core/auth/oauth.py @@ -2,6 +2,7 @@ # # @SEMANTICS: auth, oauth, oidc, adfs # @PURPOSE: ADFS OIDC configuration and client using Authlib. +# @COMPLEXITY: 2 # @LAYER: Core # @RELATION: DEPENDS_ON -> authlib # @RELATION: USES -> auth_config diff --git a/backend/src/core/auth/repository.py b/backend/src/core/auth/repository.py index be1cc1a8..880af3ce 100644 --- a/backend/src/core/auth/repository.py +++ b/backend/src/core/auth/repository.py @@ -4,12 +4,12 @@ # @SEMANTICS: auth, repository, database, user, role, permission # @PURPOSE: Data access layer for authentication and user preference entities. # @LAYER: Domain -# @RELATION: DEPENDS_ON ->[sqlalchemy.orm.Session] -# @RELATION: DEPENDS_ON ->[User:Class] -# @RELATION: DEPENDS_ON ->[Role:Class] -# @RELATION: DEPENDS_ON ->[Permission:Class] -# @RELATION: DEPENDS_ON ->[UserDashboardPreference:Class] -# @RELATION: DEPENDS_ON ->[belief_scope:Function] +# @RELATION: DEPENDS_ON -> [Session] +# @RELATION: DEPENDS_ON -> [User] +# @RELATION: DEPENDS_ON -> [Role] +# @RELATION: DEPENDS_ON -> [Permission] +# @RELATION: DEPENDS_ON -> [UserDashboardPreference] +# @RELATION: DEPENDS_ON -> [belief_scope] # @INVARIANT: All database read/write operations must execute via the injected SQLAlchemy session boundary. # @DATA_CONTRACT: Session -> [User | Role | Permission | UserDashboardPreference] # @PRE: Database connection is active. @@ -24,12 +24,13 @@ from ...models.profile import UserDashboardPreference from ..logger import belief_scope, logger # [/SECTION] + # [DEF:AuthRepository:Class] # @PURPOSE: Provides low-level CRUD operations for identity and authorization records. # @PRE: Database session is bound. # @POST: Entity instances returned safely. # @SIDE_EFFECT: Performs database reads. -# @RELATION: DEPENDS_ON -> sqlalchemy.orm.Session +# @RELATION: DEPENDS_ON -> [Session] class AuthRepository: # @PURPOSE: Initialize repository with database session. def __init__(self, db: Session): @@ -39,98 +40,125 @@ class AuthRepository: # @PURPOSE: Retrieve user by UUID. # @PRE: user_id is a valid UUID string. # @POST: Returns User object if found, else None. - # @RELATION: DEPENDS_ON -> User + # @RELATION: DEPENDS_ON -> [User] def get_user_by_id(self, user_id: str) -> Optional[User]: with belief_scope("AuthRepository.get_user_by_id"): logger.reason(f"Fetching user by id: {user_id}") result = self.db.query(User).filter(User.id == user_id).first() logger.reflect(f"User found: {result is not None}") return result + # [/DEF:get_user_by_id:Function] # [DEF:get_user_by_username:Function] # @PURPOSE: Retrieve user by username. # @PRE: username is a non-empty string. # @POST: Returns User object if found, else None. - # @RELATION: DEPENDS_ON -> User + # @RELATION: DEPENDS_ON -> [User] def get_user_by_username(self, username: str) -> Optional[User]: with belief_scope("AuthRepository.get_user_by_username"): logger.reason(f"Fetching user by username: {username}") result = self.db.query(User).filter(User.username == username).first() logger.reflect(f"User found: {result is not None}") return result + # [/DEF:get_user_by_username:Function] # [DEF:get_role_by_id:Function] # @PURPOSE: Retrieve role by UUID with permissions preloaded. - # @RELATION: DEPENDS_ON -> Role - # @RELATION: DEPENDS_ON -> Permission + # @RELATION: DEPENDS_ON -> [Role] + # @RELATION: DEPENDS_ON -> [Permission] def get_role_by_id(self, role_id: str) -> Optional[Role]: with belief_scope("AuthRepository.get_role_by_id"): - return self.db.query(Role).options(selectinload(Role.permissions)).filter(Role.id == role_id).first() + return ( + self.db.query(Role) + .options(selectinload(Role.permissions)) + .filter(Role.id == role_id) + .first() + ) + # [/DEF:get_role_by_id:Function] # [DEF:get_role_by_name:Function] # @PURPOSE: Retrieve role by unique name. - # @RELATION: DEPENDS_ON -> Role + # @RELATION: DEPENDS_ON -> [Role] def get_role_by_name(self, name: str) -> Optional[Role]: with belief_scope("AuthRepository.get_role_by_name"): return self.db.query(Role).filter(Role.name == name).first() + # [/DEF:get_role_by_name:Function] # [DEF:get_permission_by_id:Function] # @PURPOSE: Retrieve permission by UUID. - # @RELATION: DEPENDS_ON -> Permission + # @RELATION: DEPENDS_ON -> [Permission] def get_permission_by_id(self, permission_id: str) -> Optional[Permission]: with belief_scope("AuthRepository.get_permission_by_id"): - return self.db.query(Permission).filter(Permission.id == permission_id).first() + return ( + self.db.query(Permission).filter(Permission.id == permission_id).first() + ) + # [/DEF:get_permission_by_id:Function] # [DEF:get_permission_by_resource_action:Function] # @PURPOSE: Retrieve permission by resource and action tuple. - # @RELATION: DEPENDS_ON -> Permission - def get_permission_by_resource_action(self, resource: str, action: str) -> Optional[Permission]: + # @RELATION: DEPENDS_ON -> [Permission] + def get_permission_by_resource_action( + self, resource: str, action: str + ) -> Optional[Permission]: with belief_scope("AuthRepository.get_permission_by_resource_action"): - return self.db.query(Permission).filter( - Permission.resource == resource, - Permission.action == action - ).first() + return ( + self.db.query(Permission) + .filter(Permission.resource == resource, Permission.action == action) + .first() + ) + # [/DEF:get_permission_by_resource_action:Function] # [DEF:list_permissions:Function] # @PURPOSE: List all system permissions. - # @RELATION: DEPENDS_ON -> Permission + # @RELATION: DEPENDS_ON -> [Permission] def list_permissions(self) -> List[Permission]: with belief_scope("AuthRepository.list_permissions"): return self.db.query(Permission).all() + # [/DEF:list_permissions:Function] # [DEF:get_user_dashboard_preference:Function] # @PURPOSE: Retrieve dashboard filters/preferences for a user. - # @RELATION: DEPENDS_ON -> UserDashboardPreference - def get_user_dashboard_preference(self, user_id: str) -> Optional[UserDashboardPreference]: + # @RELATION: DEPENDS_ON -> [UserDashboardPreference] + def get_user_dashboard_preference( + self, user_id: str + ) -> Optional[UserDashboardPreference]: with belief_scope("AuthRepository.get_user_dashboard_preference"): - return self.db.query(UserDashboardPreference).filter( - UserDashboardPreference.user_id == user_id - ).first() + return ( + self.db.query(UserDashboardPreference) + .filter(UserDashboardPreference.user_id == user_id) + .first() + ) + # [/DEF:get_user_dashboard_preference:Function] # [DEF:get_roles_by_ad_groups:Function] # @PURPOSE: Retrieve roles that match a list of AD group names. # @PRE: groups is a list of strings representing AD group identifiers. # @POST: Returns a list of Role objects mapped to the provided AD groups. - # @RELATION: DEPENDS_ON -> Role - # @RELATION: DEPENDS_ON -> ADGroupMapping + # @RELATION: DEPENDS_ON -> [Role] + # @RELATION: DEPENDS_ON -> [ADGroupMapping] def get_roles_by_ad_groups(self, groups: List[str]) -> List[Role]: with belief_scope("AuthRepository.get_roles_by_ad_groups"): logger.reason(f"Fetching roles for AD groups: {groups}") if not groups: return [] - return self.db.query(Role).join(ADGroupMapping).filter( - ADGroupMapping.ad_group.in_(groups) - ).all() + return ( + self.db.query(Role) + .join(ADGroupMapping) + .filter(ADGroupMapping.ad_group.in_(groups)) + .all() + ) + # [/DEF:get_roles_by_ad_groups:Function] + # [/DEF:AuthRepository:Class] # [/DEF:AuthRepositoryModule:Module] diff --git a/backend/src/core/auth/security.py b/backend/src/core/auth/security.py index f04a734b..2cd48fab 100644 --- a/backend/src/core/auth/security.py +++ b/backend/src/core/auth/security.py @@ -2,6 +2,7 @@ # # @SEMANTICS: security, password, hashing, bcrypt # @PURPOSE: Utility for password hashing and verification using Passlib. +# @COMPLEXITY: 2 # @LAYER: Core # @RELATION: DEPENDS_ON -> bcrypt # diff --git a/backend/src/core/encryption_key.py b/backend/src/core/encryption_key.py index c8f0c4af..425d665c 100644 --- a/backend/src/core/encryption_key.py +++ b/backend/src/core/encryption_key.py @@ -1,10 +1,14 @@ -# [DEF:backend.src.core.encryption_key:Module] +# [DEF:EncryptionKeyModule:Module] # @COMPLEXITY: 5 # @SEMANTICS: encryption, key, bootstrap, environment, startup # @PURPOSE: Resolve and persist the Fernet encryption key required by runtime services. # @LAYER: Infra -# @RELATION: DEPENDS_ON -> backend.src.core.logger +# @RELATION: DEPENDS_ON -> [LoggerModule] # @INVARIANT: Runtime key resolution never falls back to an ephemeral secret. +# @PRE: Runtime environment can read process variables and target .env path is writable when key generation is required. +# @POST: A valid Fernet key is available to runtime services via ENCRYPTION_KEY. +# @SIDE_EFFECT: May append ENCRYPTION_KEY entry into backend .env file and set process environment variable. +# @DATA_CONTRACT: Input[env_file_path] -> Output[encryption_key] from __future__ import annotations @@ -51,6 +55,8 @@ def ensure_encryption_key(env_file_path: Path = DEFAULT_ENV_FILE_PATH) -> str: logger.reason(f"Generated ENCRYPTION_KEY and persisted it to {env_file_path}.") logger.reflect("Encryption key is available for runtime services.") return generated_key + + # [/DEF:ensure_encryption_key:Function] -# [/DEF:backend.src.core.encryption_key:Module] +# [/DEF:EncryptionKeyModule:Module] diff --git a/backend/src/core/mapping_service.py b/backend/src/core/mapping_service.py index 0e6f7741..816af9ba 100644 --- a/backend/src/core/mapping_service.py +++ b/backend/src/core/mapping_service.py @@ -1,11 +1,15 @@ -# [DEF:backend.src.core.mapping_service:Module] +# [DEF:IdMappingServiceModule:Module] # # @COMPLEXITY: 5 # @SEMANTICS: mapping, ids, synchronization, environments, cross-filters # @PURPOSE: Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID) # @LAYER: Core -# @RELATION: DEPENDS_ON -> backend.src.models.mapping (ResourceMapping, ResourceType) -# @RELATION: DEPENDS_ON -> backend.src.core.logger +# @RELATION: DEPENDS_ON -> [MappingModels] +# @RELATION: DEPENDS_ON -> [LoggerModule] +# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments. +# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution. +# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs. +# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None] # @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]} # # @INVARIANT: sync_environment must handle remote API failures gracefully. @@ -20,9 +24,17 @@ from src.models.mapping import ResourceMapping, ResourceType from src.core.logger import logger, belief_scope # [/SECTION] + # [DEF:IdMappingService:Class] # @COMPLEXITY: 5 # @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs. +# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables. +# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs. +# @RELATION: DEPENDS_ON -> [MappingModels] +# @RELATION: DEPENDS_ON -> [LoggerModule] +# @INVARIANT: self.db remains the authoritative session for all mapping operations. +# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles. +# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService] # # @TEST_CONTRACT: IdMappingServiceModel -> # { @@ -39,13 +51,13 @@ from src.core.logger import logger, belief_scope # @TEST_EDGE: get_batch_empty_list -> returns empty dict # @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure] class IdMappingService: - # [DEF:__init__:Function] # @PURPOSE: Initializes the mapping service. def __init__(self, db_session: Session): self.db = db_session self.scheduler = BackgroundScheduler() self._sync_job = None + # [/DEF:__init__:Function] # [DEF:start_scheduler:Function] @@ -53,30 +65,39 @@ class IdMappingService: # @PARAM: cron_string (str) - Cron expression for the sync interval. # @PARAM: environments (List[str]) - List of environment IDs to sync. # @PARAM: superset_client_factory - Function to get a client for an environment. - def start_scheduler(self, cron_string: str, environments: List[str], superset_client_factory): + def start_scheduler( + self, cron_string: str, environments: List[str], superset_client_factory + ): with belief_scope("IdMappingService.start_scheduler"): if self._sync_job: self.scheduler.remove_job(self._sync_job.id) - logger.info("[IdMappingService.start_scheduler][Reflect] Removed existing sync job.") - + logger.info( + "[IdMappingService.start_scheduler][Reflect] Removed existing sync job." + ) + def sync_all(): for env_id in environments: client = superset_client_factory(env_id) if client: self.sync_environment(env_id, client) - + self._sync_job = self.scheduler.add_job( sync_all, CronTrigger.from_crontab(cron_string), - id='id_mapping_sync_job', - replace_existing=True + id="id_mapping_sync_job", + replace_existing=True, ) - + if not self.scheduler.running: self.scheduler.start() - logger.info(f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}") + logger.info( + f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}" + ) else: - logger.info(f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}") + logger.info( + f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}" + ) + # [/DEF:start_scheduler:Function] # [DEF:sync_environment:Function] @@ -85,54 +106,74 @@ class IdMappingService: # @PARAM: superset_client - Instance capable of hitting the Superset API. # @PRE: environment_id exists in the database. # @POST: ResourceMapping records for the environment are created or updated. - def sync_environment(self, environment_id: str, superset_client, incremental: bool = False) -> None: + def sync_environment( + self, environment_id: str, superset_client, incremental: bool = False + ) -> None: """ Polls the Superset APIs for the target environment and updates the local mapping table. If incremental=True, only fetches items changed since the max last_synced_at date. """ with belief_scope("IdMappingService.sync_environment"): - logger.info(f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})") - + logger.info( + f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})" + ) + # Implementation Note: In a real scenario, superset_client needs to be an instance # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/ # Here we structure the logic according to the spec. - + types_to_poll = [ (ResourceType.CHART, "chart", "slice_name"), (ResourceType.DATASET, "dataset", "table_name"), - (ResourceType.DASHBOARD, "dashboard", "slug") # Note: dashboard slug or dashboard_title + ( + ResourceType.DASHBOARD, + "dashboard", + "slug", + ), # Note: dashboard slug or dashboard_title ] total_synced = 0 total_deleted = 0 try: for res_enum, endpoint, name_field in types_to_poll: - logger.debug(f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint") - + logger.debug( + f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint" + ) + # Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... ) # This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] } # We assume superset_client provides a generic method to fetch all pages. - + try: since_dttm = None if incremental: from sqlalchemy.sql import func - max_date = self.db.query(func.max(ResourceMapping.last_synced_at)).filter( - ResourceMapping.environment_id == environment_id, - ResourceMapping.resource_type == res_enum - ).scalar() - + + max_date = ( + self.db.query(func.max(ResourceMapping.last_synced_at)) + .filter( + ResourceMapping.environment_id == environment_id, + ResourceMapping.resource_type == res_enum, + ) + .scalar() + ) + if max_date: # We subtract a bit for safety overlap from datetime import timedelta - since_dttm = max_date - timedelta(minutes=5) - logger.debug(f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}") - resources = superset_client.get_all_resources(endpoint, since_dttm=since_dttm) - + since_dttm = max_date - timedelta(minutes=5) + logger.debug( + f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}" + ) + + resources = superset_client.get_all_resources( + endpoint, since_dttm=since_dttm + ) + # Track which UUIDs we see in this sync cycle synced_uuids = set() - + for res in resources: res_uuid = res.get("uuid") raw_id = res.get("id") @@ -140,16 +181,20 @@ class IdMappingService: if not res_uuid or raw_id is None: continue - + synced_uuids.add(res_uuid) - res_id = str(raw_id) # Store as string + res_id = str(raw_id) # Store as string # Upsert Logic - mapping = self.db.query(ResourceMapping).filter_by( - environment_id=environment_id, - resource_type=res_enum, - uuid=res_uuid - ).first() + mapping = ( + self.db.query(ResourceMapping) + .filter_by( + environment_id=environment_id, + resource_type=res_enum, + uuid=res_uuid, + ) + .first() + ) if mapping: mapping.remote_integer_id = res_id @@ -162,10 +207,10 @@ class IdMappingService: uuid=res_uuid, remote_integer_id=res_id, resource_name=res_name, - last_synced_at=datetime.now(timezone.utc) + last_synced_at=datetime.now(timezone.utc), ) self.db.add(new_mapping) - + total_synced += 1 # Delete stale mappings: rows for this env+type whose UUID @@ -183,19 +228,28 @@ class IdMappingService: deleted = stale_query.delete(synchronize_session="fetch") if deleted: total_deleted += deleted - logger.info(f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}") + logger.info( + f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}" + ) except Exception as loop_e: - logger.error(f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}") + logger.error( + f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}" + ) # Continue to next resource type instead of blowing up the whole sync self.db.commit() - logger.info(f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items.") - + logger.info( + f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items." + ) + except Exception as e: self.db.rollback() - logger.error(f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}") + logger.error( + f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}" + ) raise + # [/DEF:sync_environment:Function] # [DEF:get_remote_id:Function] @@ -204,19 +258,24 @@ class IdMappingService: # @PARAM: resource_type (ResourceType) # @PARAM: uuid (str) # @RETURN: Optional[int] - def get_remote_id(self, environment_id: str, resource_type: ResourceType, uuid: str) -> Optional[int]: - mapping = self.db.query(ResourceMapping).filter_by( - environment_id=environment_id, - resource_type=resource_type, - uuid=uuid - ).first() - + def get_remote_id( + self, environment_id: str, resource_type: ResourceType, uuid: str + ) -> Optional[int]: + mapping = ( + self.db.query(ResourceMapping) + .filter_by( + environment_id=environment_id, resource_type=resource_type, uuid=uuid + ) + .first() + ) + if mapping: try: return int(mapping.remote_integer_id) except ValueError: return None return None + # [/DEF:get_remote_id:Function] # [DEF:get_remote_ids_batch:Function] @@ -225,15 +284,21 @@ class IdMappingService: # @PARAM: resource_type (ResourceType) # @PARAM: uuids (List[str]) # @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID - def get_remote_ids_batch(self, environment_id: str, resource_type: ResourceType, uuids: List[str]) -> Dict[str, int]: + def get_remote_ids_batch( + self, environment_id: str, resource_type: ResourceType, uuids: List[str] + ) -> Dict[str, int]: if not uuids: return {} - mappings = self.db.query(ResourceMapping).filter( - ResourceMapping.environment_id == environment_id, - ResourceMapping.resource_type == resource_type, - ResourceMapping.uuid.in_(uuids) - ).all() + mappings = ( + self.db.query(ResourceMapping) + .filter( + ResourceMapping.environment_id == environment_id, + ResourceMapping.resource_type == resource_type, + ResourceMapping.uuid.in_(uuids), + ) + .all() + ) result = {} for m in mappings: @@ -241,9 +306,11 @@ class IdMappingService: result[m.uuid] = int(m.remote_integer_id) except ValueError: pass - + return result + # [/DEF:get_remote_ids_batch:Function] + # [/DEF:IdMappingService:Class] -# [/DEF:backend.src.core.mapping_service:Module] +# [/DEF:IdMappingServiceModule:Module] diff --git a/backend/src/core/migration/__init__.py b/backend/src/core/migration/__init__.py index 4e601c8f..bb9644c4 100644 --- a/backend/src/core/migration/__init__.py +++ b/backend/src/core/migration/__init__.py @@ -1,4 +1,4 @@ -# [DEF:backend.src.core.migration.__init__:Module] +# [DEF:MigrationPackage:Module] # @COMPLEXITY: 1 # @SEMANTICS: migration, package, exports # @PURPOSE: Namespace package for migration pre-flight orchestration components. @@ -9,4 +9,4 @@ from .archive_parser import MigrationArchiveParser __all__ = ["MigrationDryRunService", "MigrationArchiveParser"] -# [/DEF:backend.src.core.migration.__init__:Module] +# [/DEF:MigrationPackage:Module] diff --git a/backend/src/core/migration/archive_parser.py b/backend/src/core/migration/archive_parser.py index c45627f9..1d4fbb4e 100644 --- a/backend/src/core/migration/archive_parser.py +++ b/backend/src/core/migration/archive_parser.py @@ -1,9 +1,9 @@ -# [DEF:backend.src.core.migration.archive_parser:Module] +# [DEF:MigrationArchiveParserModule:Module] # @COMPLEXITY: 3 # @SEMANTICS: migration, zip, parser, yaml, metadata # @PURPOSE: Parse Superset export ZIP archives into normalized object catalogs for diffing. # @LAYER: Core -# @RELATION: DEPENDS_ON -> backend.src.core.logger +# @RELATION: DEPENDS_ON -> [LoggerModule] # @INVARIANT: Parsing is read-only and never mutates archive files. import json @@ -19,11 +19,13 @@ from ..logger import logger, belief_scope # [DEF:MigrationArchiveParser:Class] # @PURPOSE: Extract normalized dashboards/charts/datasets metadata from ZIP archives. -# @RELATION: CONTAINS -> [extract_objects_from_zip, _collect_yaml_objects, _normalize_object_payload] +# @RELATION: CONTAINS -> [extract_objects_from_zip] +# @RELATION: CONTAINS -> [_collect_yaml_objects] +# @RELATION: CONTAINS -> [_normalize_object_payload] class MigrationArchiveParser: # [DEF:extract_objects_from_zip:Function] # @PURPOSE: Extract object catalogs from Superset archive. - # @RELATION: DEPENDS_ON -> _collect_yaml_objects + # @RELATION: DEPENDS_ON -> [_collect_yaml_objects] # @PRE: zip_path points to a valid readable ZIP. # @POST: Returns object lists grouped by resource type. # @RETURN: Dict[str, List[Dict[str, Any]]] @@ -53,7 +55,7 @@ class MigrationArchiveParser: # [DEF:_collect_yaml_objects:Function] # @PURPOSE: Read and normalize YAML manifests for one object type. - # @RELATION: DEPENDS_ON -> _normalize_object_payload + # @RELATION: DEPENDS_ON -> [_normalize_object_payload] # @PRE: object_type is one of dashboards/charts/datasets. # @POST: Returns only valid normalized objects. def _collect_yaml_objects( @@ -153,4 +155,4 @@ class MigrationArchiveParser: # [/DEF:MigrationArchiveParser:Class] -# [/DEF:backend.src.core.migration.archive_parser:Module] +# [/DEF:MigrationArchiveParserModule:Module] diff --git a/backend/src/core/migration/dry_run_orchestrator.py b/backend/src/core/migration/dry_run_orchestrator.py index 58192f9a..21e79777 100644 --- a/backend/src/core/migration/dry_run_orchestrator.py +++ b/backend/src/core/migration/dry_run_orchestrator.py @@ -1,12 +1,12 @@ -# [DEF:backend.src.core.migration.dry_run_orchestrator:Module] +# [DEF:MigrationDryRunOrchestratorModule:Module] # @COMPLEXITY: 3 # @SEMANTICS: migration, dry_run, diff, risk, superset # @PURPOSE: Compute pre-flight migration diff and risk scoring without apply. # @LAYER: Core -# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client.SupersetClient] -# @RELATION: DEPENDS_ON ->[backend.src.core.migration_engine.MigrationEngine] -# @RELATION: DEPENDS_ON ->[backend.src.core.migration.archive_parser.MigrationArchiveParser] -# @RELATION: DEPENDS_ON ->[backend.src.core.migration.risk_assessor] +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RELATION: DEPENDS_ON -> [MigrationEngine] +# @RELATION: DEPENDS_ON -> [MigrationArchiveParser] +# @RELATION: DEPENDS_ON -> [RiskAssessorModule] # @INVARIANT: Dry run is informative only and must not mutate target environment. from datetime import datetime, timezone @@ -27,7 +27,14 @@ from ..utils.fileio import create_temp_file # [DEF:MigrationDryRunService:Class] # @PURPOSE: Build deterministic diff/risk payload for migration pre-flight. -# @RELATION: CONTAINS -> [__init__, run, _load_db_mapping, _accumulate_objects, _index_by_uuid, _build_object_diff, _build_target_signatures, _build_risks] +# @RELATION: CONTAINS -> [__init__] +# @RELATION: CONTAINS -> [run] +# @RELATION: CONTAINS -> [_load_db_mapping] +# @RELATION: CONTAINS -> [_accumulate_objects] +# @RELATION: CONTAINS -> [_index_by_uuid] +# @RELATION: CONTAINS -> [_build_object_diff] +# @RELATION: CONTAINS -> [_build_target_signatures] +# @RELATION: CONTAINS -> [_build_risks] class MigrationDryRunService: # [DEF:__init__:Function] # @PURPOSE: Wire parser dependency for archive object extraction. @@ -40,7 +47,11 @@ class MigrationDryRunService: # [DEF:run:Function] # @PURPOSE: Execute full dry-run computation for selected dashboards. - # @RELATION: DEPENDS_ON -> [_load_db_mapping, _accumulate_objects, _build_target_signatures, _build_object_diff, _build_risks] + # @RELATION: DEPENDS_ON -> [_load_db_mapping] + # @RELATION: DEPENDS_ON -> [_accumulate_objects] + # @RELATION: DEPENDS_ON -> [_build_target_signatures] + # @RELATION: DEPENDS_ON -> [_build_object_diff] + # @RELATION: DEPENDS_ON -> [_build_risks] # @PRE: source/target clients are authenticated and selection validated by caller. # @POST: Returns JSON-serializable pre-flight payload with summary, diff and risk. # @SIDE_EFFECT: Reads source export archives and target metadata via network. @@ -195,7 +206,7 @@ class MigrationDryRunService: # [DEF:_build_object_diff:Function] # @PURPOSE: Compute create/update/delete buckets by UUID+signature. - # @RELATION: DEPENDS_ON -> _index_by_uuid + # @RELATION: DEPENDS_ON -> [_index_by_uuid] def _build_object_diff( self, source_objects: List[Dict[str, Any]], target_objects: List[Dict[str, Any]] ) -> Dict[str, List[Dict[str, Any]]]: @@ -349,4 +360,4 @@ class MigrationDryRunService: # [/DEF:MigrationDryRunService:Class] -# [/DEF:backend.src.core.migration.dry_run_orchestrator:Module] +# [/DEF:MigrationDryRunOrchestratorModule:Module] diff --git a/backend/src/core/migration/risk_assessor.py b/backend/src/core/migration/risk_assessor.py index 9a983fab..916cdb06 100644 --- a/backend/src/core/migration/risk_assessor.py +++ b/backend/src/core/migration/risk_assessor.py @@ -1,12 +1,18 @@ -# [DEF:backend.src.core.migration.risk_assessor:Module] +# [DEF:RiskAssessorModule:Module] # @COMPLEXITY: 5 # @SEMANTICS: migration, dry_run, risk, scoring, preflight # @PURPOSE: Compute deterministic migration risk items and aggregate score for dry-run reporting. # @LAYER: Domain -# @RELATION: DEPENDS_ON -> [backend.src.core.superset_client.SupersetClient] -# @RELATION: DISPATCHED_BY -> [backend.src.core.migration.dry_run_orchestrator.MigrationDryRunService.run] -# @RELATION: CONTAINS -> [index_by_uuid, extract_owner_identifiers, build_risks, score_risks] +# @RELATION: DEPENDS_ON -> [SupersetClient] +# @RELATION: CONTAINS -> [index_by_uuid] +# @RELATION: CONTAINS -> [extract_owner_identifiers] +# @RELATION: CONTAINS -> [build_risks] +# @RELATION: CONTAINS -> [score_risks] # @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping. +# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration. +# @POST: Risk scoring output preserves item list and provides bounded score with derived level. +# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client. +# @DATA_CONTRACT: Module[build_risks, score_risks] # @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]] # @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item] # @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted] @@ -80,7 +86,8 @@ def extract_owner_identifiers(owners: Any) -> List[str]: # [DEF:build_risks:Function] # @PURPOSE: Build risk list from computed diffs and target catalog state. -# @RELATION: DEPENDS_ON -> [index_by_uuid, extract_owner_identifiers] +# @RELATION: DEPENDS_ON -> [index_by_uuid] +# @RELATION: DEPENDS_ON -> [extract_owner_identifiers] # @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures. # @PRE: target_client is authenticated/usable for database list retrieval. # @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks. @@ -192,4 +199,4 @@ def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]: # [/DEF:score_risks:Function] -# [/DEF:backend.src.core.migration.risk_assessor:Module] +# [/DEF:RiskAssessorModule:Module] diff --git a/backend/src/core/scheduler.py b/backend/src/core/scheduler.py index 29c8803b..6efdc94e 100644 --- a/backend/src/core/scheduler.py +++ b/backend/src/core/scheduler.py @@ -3,20 +3,18 @@ # @SEMANTICS: scheduler, apscheduler, cron, backup # @PURPOSE: Manages scheduled tasks using APScheduler. # @LAYER: Core -# @RELATION: Uses TaskManager to run scheduled backups. +# @RELATION: DEPENDS_ON -> TaskManager # [SECTION: IMPORTS] from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.date import DateTrigger from .logger import logger, belief_scope from .config_manager import ConfigManager -from .database import SessionLocal -from ..models.llm import ValidationPolicy import asyncio from datetime import datetime, time, timedelta, date # [/SECTION] + # [DEF:SchedulerService:Class] # @COMPLEXITY: 3 # @SEMANTICS: scheduler, service, apscheduler @@ -32,6 +30,7 @@ class SchedulerService: self.config_manager = config_manager self.scheduler = BackgroundScheduler() self.loop = asyncio.get_event_loop() + # [/DEF:__init__:Function] # [DEF:start:Function] @@ -44,6 +43,7 @@ class SchedulerService: self.scheduler.start() logger.info("Scheduler started.") self.load_schedules() + # [/DEF:start:Function] # [DEF:stop:Function] @@ -55,6 +55,7 @@ class SchedulerService: if self.scheduler.running: self.scheduler.shutdown() logger.info("Scheduler stopped.") + # [/DEF:stop:Function] # [DEF:load_schedules:Function] @@ -65,11 +66,12 @@ class SchedulerService: with belief_scope("SchedulerService.load_schedules"): # Clear existing jobs self.scheduler.remove_all_jobs() - + config = self.config_manager.get_config() for env in config.environments: if env.backup_schedule and env.backup_schedule.enabled: self.add_backup_job(env.id, env.backup_schedule.cron_expression) + # [/DEF:load_schedules:Function] # [DEF:add_backup_job:Function] @@ -79,7 +81,10 @@ class SchedulerService: # @PARAM: env_id (str) - The ID of the environment. # @PARAM: cron_expression (str) - The cron expression for the schedule. def add_backup_job(self, env_id: str, cron_expression: str): - with belief_scope("SchedulerService.add_backup_job", f"env_id={env_id}, cron={cron_expression}"): + with belief_scope( + "SchedulerService.add_backup_job", + f"env_id={env_id}, cron={cron_expression}", + ): job_id = f"backup_{env_id}" try: self.scheduler.add_job( @@ -87,11 +92,14 @@ class SchedulerService: CronTrigger.from_crontab(cron_expression), id=job_id, args=[env_id], - replace_existing=True + replace_existing=True, + ) + logger.info( + f"Scheduled backup job added for environment {env_id}: {cron_expression}" ) - logger.info(f"Scheduled backup job added for environment {env_id}: {cron_expression}") except Exception as e: logger.error(f"Failed to add backup job for environment {env_id}: {e}") + # [/DEF:add_backup_job:Function] # [DEF:_trigger_backup:Function] @@ -102,30 +110,45 @@ class SchedulerService: def _trigger_backup(self, env_id: str): with belief_scope("SchedulerService._trigger_backup", f"env_id={env_id}"): logger.info(f"Triggering scheduled backup for environment {env_id}") - + # Check if a backup is already running for this environment active_tasks = self.task_manager.get_tasks(limit=100) for task in active_tasks: - if (task.plugin_id == "superset-backup" and - task.status in ["PENDING", "RUNNING"] and - task.params.get("environment_id") == env_id): - logger.warning(f"Backup already running for environment {env_id}. Skipping scheduled run.") + if ( + task.plugin_id == "superset-backup" + and task.status in ["PENDING", "RUNNING"] + and task.params.get("environment_id") == env_id + ): + logger.warning( + f"Backup already running for environment {env_id}. Skipping scheduled run." + ) return # Run the backup task # We need to run this in the event loop since create_task is async asyncio.run_coroutine_threadsafe( - self.task_manager.create_task("superset-backup", {"environment_id": env_id}), - self.loop + self.task_manager.create_task( + "superset-backup", {"environment_id": env_id} + ), + self.loop, ) + # [/DEF:_trigger_backup:Function] + # [/DEF:SchedulerService:Class] + # [DEF:ThrottledSchedulerConfigurator:Class] # @COMPLEXITY: 5 # @SEMANTICS: scheduler, throttling, distribution # @PURPOSE: Distributes validation tasks evenly within an execution window. +# @PRE: Validation policies provide a finite dashboard list and a valid execution window. +# @POST: Produces deterministic per-dashboard run timestamps within the configured window. +# @RELATION: DEPENDS_ON -> SchedulerModule +# @INVARIANT: Returned schedule size always matches number of dashboard IDs. +# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows. +# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]] class ThrottledSchedulerConfigurator: # [DEF:calculate_schedule:Function] # @PURPOSE: Calculates execution times for N tasks within a window. @@ -134,10 +157,7 @@ class ThrottledSchedulerConfigurator: # @INVARIANT: Tasks are distributed with near-even spacing. @staticmethod def calculate_schedule( - window_start: time, - window_end: time, - dashboard_ids: list, - current_date: date + window_start: time, window_end: time, dashboard_ids: list, current_date: date ) -> list: with belief_scope("ThrottledSchedulerConfigurator.calculate_schedule"): n = len(dashboard_ids) @@ -152,32 +172,39 @@ class ThrottledSchedulerConfigurator: end_dt += timedelta(days=1) total_seconds = (end_dt - start_dt).total_seconds() - + # Minimum interval of 1 second to avoid division by zero or negative if total_seconds <= 0: - logger.warning(f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks.") + logger.warning( + f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks." + ) return [start_dt] * n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds), # we still distribute them but they might be very close. # The requirement says "near-even spacing". - + if n == 1: return [start_dt] interval = total_seconds / (n - 1) if n > 1 else 0 - + # If interval is too small (e.g. < 1s), we might want a fallback, # but the spec says "handle too-small windows with explicit fallback/warning". if interval < 1: - logger.warning(f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated.") + logger.warning( + f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated." + ) scheduled_times = [] for i in range(n): scheduled_times.append(start_dt + timedelta(seconds=i * interval)) - + return scheduled_times + # [/DEF:calculate_schedule:Function] + + # [/DEF:ThrottledSchedulerConfigurator:Class] -# [/DEF:SchedulerModule:Module] \ No newline at end of file +# [/DEF:SchedulerModule:Module] diff --git a/backend/src/core/task_manager/context.py b/backend/src/core/task_manager/context.py index 3e410946..45f3104c 100644 --- a/backend/src/core/task_manager/context.py +++ b/backend/src/core/task_manager/context.py @@ -2,9 +2,14 @@ # @SEMANTICS: task, context, plugin, execution, logger # @PURPOSE: Provides execution context passed to plugins during task execution. # @LAYER: Core -# @RELATION: DEPENDS_ON -> TaskLogger, USED_BY -> plugins +# @RELATION: DEPENDS_ON -> [TaskLoggerModule] +# @RELATION: USED_BY -> [TaskManager] # @COMPLEXITY: 5 # @INVARIANT: Each TaskContext is bound to a single task execution. +# @PRE: Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries. +# @POST: Plugins receive context instances with stable logger and parameter accessors. +# @SIDE_EFFECT: Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts. +# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext] # [SECTION: IMPORTS] # [SECTION: IMPORTS] @@ -13,11 +18,17 @@ from .task_logger import TaskLogger from ..logger import belief_scope # [/SECTION] + # [DEF:TaskContext:Class] # @SEMANTICS: context, task, execution, plugin # @PURPOSE: A container passed to plugin.execute() providing the logger and other task-specific utilities. # @COMPLEXITY: 5 # @INVARIANT: logger is always a valid TaskLogger instance. +# @PRE: Constructor receives non-empty task_id, callable add_log_fn, and params mapping. +# @POST: Instance exposes immutable task identity with logger, params, and optional background task access. +# @RELATION: DEPENDS_ON -> [TaskLogger] +# @SIDE_EFFECT: Emits structured task logs through TaskLogger on plugin interactions. +# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext] # @UX_STATE: Idle -> Active -> Complete # # @TEST_CONTRACT: TaskContextInit -> @@ -36,7 +47,7 @@ from ..logger import belief_scope class TaskContext: """ Execution context provided to plugins during task execution. - + Usage: def execute(params: dict, context: TaskContext = None): if context: @@ -44,7 +55,7 @@ class TaskContext: context.logger.progress("Processing items", percent=50) # ... plugin logic """ - + # [DEF:__init__:Function] # @PURPOSE: Initialize the TaskContext with task-specific resources. # @PRE: task_id is a valid task identifier, add_log_fn is callable. @@ -66,12 +77,11 @@ class TaskContext: self._params = params self._background_tasks = background_tasks self._logger = TaskLogger( - task_id=task_id, - add_log_fn=add_log_fn, - source=default_source + task_id=task_id, add_log_fn=add_log_fn, source=default_source ) + # [/DEF:__init__:Function] - + # [DEF:task_id:Function] # @PURPOSE: Get the task ID. # @PRE: TaskContext must be initialized. @@ -81,8 +91,9 @@ class TaskContext: def task_id(self) -> str: with belief_scope("task_id"): return self._task_id + # [/DEF:task_id:Function] - + # [DEF:logger:Function] # @PURPOSE: Get the TaskLogger instance for this context. # @PRE: TaskContext must be initialized. @@ -92,8 +103,9 @@ class TaskContext: def logger(self) -> TaskLogger: with belief_scope("logger"): return self._logger + # [/DEF:logger:Function] - + # [DEF:params:Function] # @PURPOSE: Get the task parameters. # @PRE: TaskContext must be initialized. @@ -103,6 +115,7 @@ class TaskContext: def params(self) -> Dict[str, Any]: with belief_scope("params"): return self._params + # [/DEF:params:Function] # [DEF:background_tasks:Function] @@ -113,8 +126,9 @@ class TaskContext: def background_tasks(self) -> Optional[Any]: with belief_scope("background_tasks"): return self._background_tasks + # [/DEF:background_tasks:Function] - + # [DEF:get_param:Function] # @PURPOSE: Get a specific parameter value with optional default. # @PRE: TaskContext must be initialized. @@ -125,8 +139,9 @@ class TaskContext: def get_param(self, key: str, default: Any = None) -> Any: with belief_scope("get_param"): return self._params.get(key, default) + # [/DEF:get_param:Function] - + # [DEF:create_sub_context:Function] # @PURPOSE: Create a sub-context with a different default source. # @PRE: source is a non-empty string. @@ -143,8 +158,10 @@ class TaskContext: default_source=source, background_tasks=self._background_tasks, ) + # [/DEF:create_sub_context:Function] + # [/DEF:TaskContext:Class] # [/DEF:TaskContextModule:Module] diff --git a/backend/src/core/task_manager/manager.py b/backend/src/core/task_manager/manager.py index c6317618..b2c48a74 100644 --- a/backend/src/core/task_manager/manager.py +++ b/backend/src/core/task_manager/manager.py @@ -8,8 +8,9 @@ # @POST: Orchestrates task execution and persistence. # @SIDE_EFFECT: Spawns worker threads and flushes logs to DB. # @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry] -# @RELATION: [DEPENDS_ON] ->[PluginLoader:Class] -# @RELATION: [DEPENDS_ON] ->[TaskPersistenceModule:Module] +# @RELATION: [DEPENDS_ON] ->[PluginLoader] +# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService] +# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService] # @INVARIANT: Task IDs are unique. # @CONSTRAINT: Must use belief_scope for logging. # @TEST_CONTRACT: TaskManagerModule -> { @@ -38,15 +39,16 @@ from .context import TaskContext from ..logger import logger, belief_scope, should_log_task_level # [/SECTION] + # [DEF:TaskManager:Class] # @TIER: CRITICAL # @COMPLEXITY: 5 # @SEMANTICS: task, manager, lifecycle, execution, state # @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. # @LAYER: Core -# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService:Class] -# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService:Class] -# @RELATION: [DEPENDS_ON] ->[PluginLoader:Class] +# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService] +# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService] +# @RELATION: [DEPENDS_ON] ->[PluginLoader] # @INVARIANT: Task IDs are unique within the registry. # @INVARIANT: Each task has exactly one status at any time. # @INVARIANT: Log entries are never deleted after being added to a task. @@ -56,42 +58,49 @@ class TaskManager: """ Manages the lifecycle of tasks, including their creation, execution, and state tracking. """ - + # Log flush interval in seconds LOG_FLUSH_INTERVAL = 2.0 - + # [DEF:__init__:Function] # @COMPLEXITY: 5 # @PURPOSE: Initialize the TaskManager with dependencies. # @PRE: plugin_loader is initialized. # @POST: TaskManager is ready to accept tasks. + # @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory. + # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks] # @PARAM: plugin_loader - The plugin loader instance. def __init__(self, plugin_loader): with belief_scope("TaskManager.__init__"): self.plugin_loader = plugin_loader self.tasks: Dict[str, Task] = {} self.subscribers: Dict[str, List[asyncio.Queue]] = {} - self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution + self.executor = ThreadPoolExecutor( + max_workers=5 + ) # For CPU-bound plugin execution self.persistence_service = TaskPersistenceService() self.log_persistence_service = TaskLogPersistenceService() - + # Log buffer: task_id -> List[LogEntry] self._log_buffer: Dict[str, List[LogEntry]] = {} self._log_buffer_lock = threading.Lock() - + # Flusher thread for batch writing logs self._flusher_stop_event = threading.Event() - self._flusher_thread = threading.Thread(target=self._flusher_loop, daemon=True) + self._flusher_thread = threading.Thread( + target=self._flusher_loop, daemon=True + ) self._flusher_thread.start() - + try: self.loop = asyncio.get_running_loop() except RuntimeError: self.loop = asyncio.get_event_loop() self.task_futures: Dict[str, asyncio.Future] = {} - + # Load persisted tasks on startup self.load_persisted_tasks() + # [/DEF:__init__:Function] # [DEF:_flusher_loop:Function] @@ -99,11 +108,13 @@ class TaskManager: # @PURPOSE: Background thread that periodically flushes log buffer to database. # @PRE: TaskManager is initialized. # @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds. + # @RELATION: [CALLS] ->[TaskManager._flush_logs] def _flusher_loop(self): """Background thread that flushes log buffer to database.""" while not self._flusher_stop_event.is_set(): self._flush_logs() self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL) + # [/DEF:_flusher_loop:Function] # [DEF:_flush_logs:Function] @@ -111,15 +122,16 @@ class TaskManager: # @PURPOSE: Flush all buffered logs to the database. # @PRE: None. # @POST: All buffered logs are written to task_logs table. + # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs] def _flush_logs(self): """Flush all buffered logs to the database.""" with self._log_buffer_lock: task_ids = list(self._log_buffer.keys()) - + for task_id in task_ids: with self._log_buffer_lock: logs = self._log_buffer.pop(task_id, []) - + if logs: try: self.log_persistence_service.add_logs(task_id, logs) @@ -131,6 +143,7 @@ class TaskManager: if task_id not in self._log_buffer: self._log_buffer[task_id] = [] self._log_buffer[task_id].extend(logs) + # [/DEF:_flush_logs:Function] # [DEF:_flush_task_logs:Function] @@ -139,17 +152,19 @@ class TaskManager: # @PRE: task_id exists. # @POST: Task's buffered logs are written to database. # @PARAM: task_id (str) - The task ID. + # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs] def _flush_task_logs(self, task_id: str): """Flush logs for a specific task immediately.""" with belief_scope("_flush_task_logs"): with self._log_buffer_lock: logs = self._log_buffer.pop(task_id, []) - + if logs: try: self.log_persistence_service.add_logs(task_id, logs) except Exception as e: logger.error(f"Failed to flush logs for task {task_id}: {e}") + # [/DEF:_flush_task_logs:Function] # [DEF:create_task:Function] @@ -162,24 +177,30 @@ class TaskManager: # @PARAM: user_id (Optional[str]) - ID of the user requesting the task. # @RETURN: Task - The created task instance. # @THROWS: ValueError if plugin not found or params invalid. - async def create_task(self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None) -> Task: + # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] + async def create_task( + self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None + ) -> Task: with belief_scope("TaskManager.create_task", f"plugin_id={plugin_id}"): if not self.plugin_loader.has_plugin(plugin_id): logger.error(f"Plugin with ID '{plugin_id}' not found.") raise ValueError(f"Plugin with ID '{plugin_id}' not found.") self.plugin_loader.get_plugin(plugin_id) - + if not isinstance(params, dict): - logger.error("Task parameters must be a dictionary.") - raise ValueError("Task parameters must be a dictionary.") + logger.error("Task parameters must be a dictionary.") + raise ValueError("Task parameters must be a dictionary.") task = Task(plugin_id=plugin_id, params=params, user_id=user_id) self.tasks[task.id] = task self.persistence_service.persist_task(task) logger.info(f"Task {task.id} created and scheduled for execution") - self.loop.create_task(self._run_task(task.id)) # Schedule task for execution + self.loop.create_task( + self._run_task(task.id) + ) # Schedule task for execution return task + # [/DEF:create_task:Function] # [DEF:_run_task:Function] @@ -188,25 +209,33 @@ class TaskManager: # @PRE: Task exists in registry. # @POST: Task is executed, status updated to SUCCESS or FAILED. # @PARAM: task_id (str) - The ID of the task to run. + # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] async def _run_task(self, task_id: str): with belief_scope("TaskManager._run_task", f"task_id={task_id}"): task = self.tasks[task_id] plugin = self.plugin_loader.get_plugin(task.plugin_id) - logger.info(f"Starting execution of task {task_id} for plugin '{plugin.name}'") + logger.info( + f"Starting execution of task {task_id} for plugin '{plugin.name}'" + ) task.status = TaskStatus.RUNNING task.started_at = datetime.utcnow() self.persistence_service.persist_task(task) - self._add_log(task_id, "INFO", f"Task started for plugin '{plugin.name}'", source="system") + self._add_log( + task_id, + "INFO", + f"Task started for plugin '{plugin.name}'", + source="system", + ) try: # Prepare params and check if plugin supports new TaskContext params = {**task.params, "_task_id": task_id} - + # Check if plugin's execute method accepts 'context' parameter sig = inspect.signature(plugin.execute) - accepts_context = 'context' in sig.parameters - + accepts_context = "context" in sig.parameters + if accepts_context: # Create TaskContext for new-style plugins context = TaskContext( @@ -216,13 +245,13 @@ class TaskManager: default_source="plugin", background_tasks=None, ) - + if asyncio.iscoroutinefunction(plugin.execute): task.result = await plugin.execute(params, context=context) else: task.result = await self.loop.run_in_executor( self.executor, - lambda: plugin.execute(params, context=context) + lambda: plugin.execute(params, context=context), ) else: # Backward compatibility: old-style plugins without context @@ -230,24 +259,36 @@ class TaskManager: task.result = await plugin.execute(params) else: task.result = await self.loop.run_in_executor( - self.executor, - plugin.execute, - params + self.executor, plugin.execute, params ) - + logger.info(f"Task {task_id} completed successfully") task.status = TaskStatus.SUCCESS - self._add_log(task_id, "INFO", f"Task completed successfully for plugin '{plugin.name}'", source="system") + self._add_log( + task_id, + "INFO", + f"Task completed successfully for plugin '{plugin.name}'", + source="system", + ) except Exception as e: logger.error(f"Task {task_id} failed: {e}") task.status = TaskStatus.FAILED - self._add_log(task_id, "ERROR", f"Task failed: {e}", source="system", metadata={"error_type": type(e).__name__}) + self._add_log( + task_id, + "ERROR", + f"Task failed: {e}", + source="system", + metadata={"error_type": type(e).__name__}, + ) finally: task.finished_at = datetime.utcnow() # Flush any remaining buffered logs before persisting task self._flush_task_logs(task_id) self.persistence_service.persist_task(task) - logger.info(f"Task {task_id} execution finished with status: {task.status}") + logger.info( + f"Task {task_id} execution finished with status: {task.status}" + ) + # [/DEF:_run_task:Function] # [DEF:resolve_task:Function] @@ -258,21 +299,23 @@ class TaskManager: # @PARAM: task_id (str) - The ID of the task. # @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait. # @THROWS: ValueError if task not found or not awaiting mapping. + # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]): with belief_scope("TaskManager.resolve_task", f"task_id={task_id}"): task = self.tasks.get(task_id) if not task or task.status != TaskStatus.AWAITING_MAPPING: raise ValueError("Task is not awaiting mapping.") - + # Update task params with resolution task.params.update(resolution_params) task.status = TaskStatus.RUNNING self.persistence_service.persist_task(task) self._add_log(task_id, "INFO", "Task resumed after mapping resolution.") - + # Signal the future to continue if task_id in self.task_futures: self.task_futures[task_id].set_result(True) + # [/DEF:resolve_task:Function] # [DEF:wait_for_resolution:Function] @@ -281,21 +324,23 @@ class TaskManager: # @PRE: Task exists. # @POST: Execution pauses until future is set. # @PARAM: task_id (str) - The ID of the task. + # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] async def wait_for_resolution(self, task_id: str): with belief_scope("TaskManager.wait_for_resolution", f"task_id={task_id}"): task = self.tasks.get(task_id) if not task: return - + task.status = TaskStatus.AWAITING_MAPPING self.persistence_service.persist_task(task) self.task_futures[task_id] = self.loop.create_future() - + try: await self.task_futures[task_id] finally: if task_id in self.task_futures: del self.task_futures[task_id] + # [/DEF:wait_for_resolution:Function] # [DEF:wait_for_input:Function] @@ -304,20 +349,22 @@ class TaskManager: # @PRE: Task exists. # @POST: Execution pauses until future is set via resume_task_with_password. # @PARAM: task_id (str) - The ID of the task. + # @RELATION: [CALLS] ->[asyncio.AbstractEventLoop.create_future] async def wait_for_input(self, task_id: str): with belief_scope("TaskManager.wait_for_input", f"task_id={task_id}"): task = self.tasks.get(task_id) if not task: return - + # Status is already set to AWAITING_INPUT by await_input() self.task_futures[task_id] = self.loop.create_future() - + try: await self.task_futures[task_id] finally: if task_id in self.task_futures: del self.task_futures[task_id] + # [/DEF:wait_for_input:Function] # [DEF:get_task:Function] @@ -327,9 +374,11 @@ class TaskManager: # @POST: Returns Task object or None. # @PARAM: task_id (str) - ID of the task. # @RETURN: Optional[Task] - The task or None. + # @RELATION: [READS] ->[TaskManager.tasks] def get_task(self, task_id: str) -> Optional[Task]: with belief_scope("TaskManager.get_task", f"task_id={task_id}"): return self.tasks.get(task_id) + # [/DEF:get_task:Function] # [DEF:get_all_tasks:Function] @@ -338,9 +387,11 @@ class TaskManager: # @PRE: None. # @POST: Returns list of all Task objects. # @RETURN: List[Task] - All tasks. + # @RELATION: [READS] ->[TaskManager.tasks] def get_all_tasks(self) -> List[Task]: with belief_scope("TaskManager.get_all_tasks"): return list(self.tasks.values()) + # [/DEF:get_all_tasks:Function] # [DEF:get_tasks:Function] @@ -352,13 +403,14 @@ class TaskManager: # @PARAM: offset (int) - Number of tasks to skip. # @PARAM: status (Optional[TaskStatus]) - Filter by task status. # @RETURN: List[Task] - List of tasks matching criteria. + # @RELATION: [READS] ->[TaskManager.tasks] def get_tasks( self, limit: int = 10, offset: int = 0, status: Optional[TaskStatus] = None, plugin_ids: Optional[List[str]] = None, - completed_only: bool = False + completed_only: bool = False, ) -> List[Task]: with belief_scope("TaskManager.get_tasks"): tasks = list(self.tasks.values()) @@ -368,7 +420,10 @@ class TaskManager: plugin_id_set = set(plugin_ids) tasks = [t for t in tasks if t.plugin_id in plugin_id_set] if completed_only: - tasks = [t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]] + tasks = [ + t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED] + ] + # Sort by started_at descending with tolerant handling of mixed tz-aware/naive values. def sort_key(task: Task) -> float: started_at = task.started_at @@ -381,7 +436,8 @@ class TaskManager: return started_at.timestamp() tasks.sort(key=sort_key, reverse=True) - return tasks[offset:offset + limit] + return tasks[offset : offset + limit] + # [/DEF:get_tasks:Function] # [DEF:get_task_logs:Function] @@ -392,10 +448,13 @@ class TaskManager: # @PARAM: task_id (str) - ID of the task. # @PARAM: log_filter (Optional[LogFilter]) - Filter parameters. # @RETURN: List[LogEntry] - List of log entries. - def get_task_logs(self, task_id: str, log_filter: Optional[LogFilter] = None) -> List[LogEntry]: + # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs] + def get_task_logs( + self, task_id: str, log_filter: Optional[LogFilter] = None + ) -> List[LogEntry]: with belief_scope("TaskManager.get_task_logs", f"task_id={task_id}"): task = self.tasks.get(task_id) - + # For completed tasks, fetch from persistence if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]: if log_filter is None: @@ -408,15 +467,16 @@ class TaskManager: level=log.level, message=log.message, source=log.source, - metadata=log.metadata + metadata=log.metadata, ) for log in task_logs ] - + # For running/pending tasks, return from memory return task.logs if task else [] + # [/DEF:get_task_logs:Function] - + # [DEF:get_task_log_stats:Function] # @COMPLEXITY: 3 # @PURPOSE: Get statistics about logs for a task. @@ -424,11 +484,13 @@ class TaskManager: # @POST: Returns LogStats with counts by level and source. # @PARAM: task_id (str) - The task ID. # @RETURN: LogStats - Statistics about task logs. + # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats] def get_task_log_stats(self, task_id: str) -> LogStats: with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"): return self.log_persistence_service.get_log_stats(task_id) + # [/DEF:get_task_log_stats:Function] - + # [DEF:get_task_log_sources:Function] # @COMPLEXITY: 3 # @PURPOSE: Get unique sources for a task's logs. @@ -436,9 +498,11 @@ class TaskManager: # @POST: Returns list of unique source strings. # @PARAM: task_id (str) - The task ID. # @RETURN: List[str] - Unique source names. + # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources] def get_task_log_sources(self, task_id: str) -> List[str]: with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"): return self.log_persistence_service.get_sources(task_id) + # [/DEF:get_task_log_sources:Function] # [DEF:_add_log:Function] @@ -452,6 +516,7 @@ class TaskManager: # @PARAM: source (str) - Source component (default: "system"). # @PARAM: metadata (Optional[Dict]) - Additional structured data. # @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility). + # @RELATION: [CALLS] ->[should_log_task_level] def _add_log( self, task_id: str, @@ -459,7 +524,7 @@ class TaskManager: message: str, source: str = "system", metadata: Optional[Dict[str, Any]] = None, - context: Optional[Dict[str, Any]] = None + context: Optional[Dict[str, Any]] = None, ): with belief_scope("TaskManager._add_log", f"task_id={task_id}"): task = self.tasks.get(task_id) @@ -476,12 +541,12 @@ class TaskManager: message=message, source=source, metadata=metadata, - context=context # Keep for backward compatibility + context=context, # Keep for backward compatibility ) - + # Add to in-memory logs (for backward compatibility with legacy JSON field) task.logs.append(log_entry) - + # Add to buffer for batch persistence with self._log_buffer_lock: if task_id not in self._log_buffer: @@ -492,6 +557,7 @@ class TaskManager: if task_id in self.subscribers: for queue in self.subscribers[task_id]: self.loop.call_soon_threadsafe(queue.put_nowait, log_entry) + # [/DEF:_add_log:Function] # [DEF:subscribe_logs:Function] @@ -501,6 +567,7 @@ class TaskManager: # @POST: Returns an asyncio.Queue for log entries. # @PARAM: task_id (str) - ID of the task. # @RETURN: asyncio.Queue - Queue for log entries. + # @RELATION: [MUTATES] ->[TaskManager.subscribers] async def subscribe_logs(self, task_id: str) -> asyncio.Queue: with belief_scope("TaskManager.subscribe_logs", f"task_id={task_id}"): queue = asyncio.Queue() @@ -508,6 +575,7 @@ class TaskManager: self.subscribers[task_id] = [] self.subscribers[task_id].append(queue) return queue + # [/DEF:subscribe_logs:Function] # [DEF:unsubscribe_logs:Function] @@ -517,6 +585,7 @@ class TaskManager: # @POST: Queue removed from subscribers. # @PARAM: task_id (str) - ID of the task. # @PARAM: queue (asyncio.Queue) - Queue to remove. + # @RELATION: [MUTATES] ->[TaskManager.subscribers] def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue): with belief_scope("TaskManager.unsubscribe_logs", f"task_id={task_id}"): if task_id in self.subscribers: @@ -524,6 +593,7 @@ class TaskManager: self.subscribers[task_id].remove(queue) if not self.subscribers[task_id]: del self.subscribers[task_id] + # [/DEF:unsubscribe_logs:Function] # [DEF:load_persisted_tasks:Function] @@ -531,12 +601,14 @@ class TaskManager: # @PURPOSE: Load persisted tasks using persistence service. # @PRE: None. # @POST: Persisted tasks loaded into self.tasks. + # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks] def load_persisted_tasks(self) -> None: with belief_scope("TaskManager.load_persisted_tasks"): loaded_tasks = self.persistence_service.load_tasks(limit=100) for task in loaded_tasks: if task.id not in self.tasks: self.tasks[task.id] = task + # [/DEF:load_persisted_tasks:Function] # [DEF:await_input:Function] @@ -547,19 +619,28 @@ class TaskManager: # @PARAM: task_id (str) - ID of the task. # @PARAM: input_request (Dict) - Details about required input. # @THROWS: ValueError if task not found or not RUNNING. + # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None: with belief_scope("TaskManager.await_input", f"task_id={task_id}"): task = self.tasks.get(task_id) if not task: raise ValueError(f"Task {task_id} not found") if task.status != TaskStatus.RUNNING: - raise ValueError(f"Task {task_id} is not RUNNING (current: {task.status})") - + raise ValueError( + f"Task {task_id} is not RUNNING (current: {task.status})" + ) + task.status = TaskStatus.AWAITING_INPUT task.input_required = True task.input_request = input_request self.persistence_service.persist_task(task) - self._add_log(task_id, "INFO", "Task paused for user input", {"input_request": input_request}) + self._add_log( + task_id, + "INFO", + "Task paused for user input", + {"input_request": input_request}, + ) + # [/DEF:await_input:Function] # [DEF:resume_task_with_password:Function] @@ -570,26 +651,39 @@ class TaskManager: # @PARAM: task_id (str) - ID of the task. # @PARAM: passwords (Dict[str, str]) - Mapping of database name to password. # @THROWS: ValueError if task not found, not awaiting input, or passwords invalid. - def resume_task_with_password(self, task_id: str, passwords: Dict[str, str]) -> None: - with belief_scope("TaskManager.resume_task_with_password", f"task_id={task_id}"): + # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] + def resume_task_with_password( + self, task_id: str, passwords: Dict[str, str] + ) -> None: + with belief_scope( + "TaskManager.resume_task_with_password", f"task_id={task_id}" + ): task = self.tasks.get(task_id) if not task: raise ValueError(f"Task {task_id} not found") if task.status != TaskStatus.AWAITING_INPUT: - raise ValueError(f"Task {task_id} is not AWAITING_INPUT (current: {task.status})") - + raise ValueError( + f"Task {task_id} is not AWAITING_INPUT (current: {task.status})" + ) + if not isinstance(passwords, dict) or not passwords: raise ValueError("Passwords must be a non-empty dictionary") - + task.params["passwords"] = passwords task.input_required = False task.input_request = None task.status = TaskStatus.RUNNING self.persistence_service.persist_task(task) - self._add_log(task_id, "INFO", "Task resumed with passwords", {"databases": list(passwords.keys())}) - + self._add_log( + task_id, + "INFO", + "Task resumed with passwords", + {"databases": list(passwords.keys())}, + ) + if task_id in self.task_futures: self.task_futures[task_id].set_result(True) + # [/DEF:resume_task_with_password:Function] # [DEF:clear_tasks:Function] @@ -599,6 +693,7 @@ class TaskManager: # @POST: Tasks matching filter (or all non-active) cleared from registry and database. # @PARAM: status (Optional[TaskStatus]) - Filter by task status. # @RETURN: int - Number of tasks cleared. + # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks] def clear_tasks(self, status: Optional[TaskStatus] = None) -> int: with belief_scope("TaskManager.clear_tasks"): tasks_to_remove = [] @@ -607,16 +702,20 @@ class TaskManager: # If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?) # Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum. # RUNNING is active execution. - + should_remove = False if status: if task.status == status: should_remove = True else: # Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING) - if task.status not in [TaskStatus.RUNNING, TaskStatus.AWAITING_INPUT, TaskStatus.AWAITING_MAPPING]: + if task.status not in [ + TaskStatus.RUNNING, + TaskStatus.AWAITING_INPUT, + TaskStatus.AWAITING_MAPPING, + ]: should_remove = True - + if should_remove: tasks_to_remove.append(task_id) @@ -625,18 +724,21 @@ class TaskManager: if tid in self.task_futures: self.task_futures[tid].cancel() del self.task_futures[tid] - + del self.tasks[tid] # Remove from persistence (task_records and task_logs via CASCADE) self.persistence_service.delete_tasks(tasks_to_remove) - + # Also explicitly delete logs (in case CASCADE is not set up) if tasks_to_remove: self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove) - + logger.info(f"Cleared {len(tasks_to_remove)} tasks.") return len(tasks_to_remove) + # [/DEF:clear_tasks:Function] + + # [/DEF:TaskManager:Class] # [/DEF:TaskManager:Module] diff --git a/backend/src/core/task_manager/models.py b/backend/src/core/task_manager/models.py index fe32efd8..1c756b88 100644 --- a/backend/src/core/task_manager/models.py +++ b/backend/src/core/task_manager/models.py @@ -3,7 +3,8 @@ # @SEMANTICS: task, models, pydantic, enum, state # @PURPOSE: Defines the data models and enumerations used by the Task Manager. # @LAYER: Core -# @RELATION: Used by TaskManager and API routes. +# @RELATION: [USED_BY] -> [TaskManager] +# @RELATION: [USED_BY] -> [ApiRoutes] # @INVARIANT: Task IDs are immutable once created. # @CONSTRAINT: Must use Pydantic for data validation. @@ -16,6 +17,7 @@ from typing import Dict, Any, List, Optional from pydantic import BaseModel, Field # [/SECTION] + # [DEF:TaskStatus:Enum] # @COMPLEXITY: 1 # @SEMANTICS: task, status, state, enum @@ -27,23 +29,33 @@ class TaskStatus(str, Enum): FAILED = "FAILED" AWAITING_MAPPING = "AWAITING_MAPPING" AWAITING_INPUT = "AWAITING_INPUT" + + # [/DEF:TaskStatus:Enum] + # [DEF:LogLevel:Enum] # @SEMANTICS: log, level, severity, enum # @PURPOSE: Defines the possible log levels for task logging. -# @COMPLEXITY: 3 +# @COMPLEXITY: 1 +# @RELATION: [USED_BY] -> [LogEntry] +# @RELATION: [USED_BY] -> [TaskLogger] class LogLevel(str, Enum): DEBUG = "DEBUG" INFO = "INFO" WARNING = "WARNING" ERROR = "ERROR" + + # [/DEF:LogLevel:Enum] + # [DEF:LogEntry:Class] # @SEMANTICS: log, entry, record, pydantic # @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task. -# @COMPLEXITY: 5 +# @COMPLEXITY: 2 +# @RELATION: [DEPENDS_ON] -> [LogLevel] +# @RELATION: [PART_OF] -> [Task] # @INVARIANT: Each log entry has a unique timestamp and source. # # @TEST_CONTRACT: LogEntryModel -> @@ -57,16 +69,25 @@ class LogEntry(BaseModel): timestamp: datetime = Field(default_factory=datetime.utcnow) level: str = Field(default="INFO") message: str - source: str = Field(default="system") # Component attribution: plugin, superset_api, git, etc. - context: Optional[Dict[str, Any]] = None # Legacy field, kept for backward compatibility - metadata: Optional[Dict[str, Any]] = None # Structured metadata (e.g., dashboard_id, progress) + source: str = Field( + default="system" + ) # Component attribution: plugin, superset_api, git, etc. + context: Optional[Dict[str, Any]] = ( + None # Legacy field, kept for backward compatibility + ) + metadata: Optional[Dict[str, Any]] = ( + None # Structured metadata (e.g., dashboard_id, progress) + ) + + # [/DEF:LogEntry:Class] + # [DEF:TaskLog:Class] # @SEMANTICS: task, log, persistent, pydantic # @PURPOSE: A Pydantic model representing a persisted log entry from the database. # @COMPLEXITY: 3 -# @RELATION: MAPS_TO -> TaskLogRecord +# @RELATION: [MAPS_TO] -> [TaskLogRecord] class TaskLog(BaseModel): id: int task_id: str @@ -78,34 +99,48 @@ class TaskLog(BaseModel): class Config: from_attributes = True + + # [/DEF:TaskLog:Class] + # [DEF:LogFilter:Class] # @SEMANTICS: log, filter, query, pydantic # @PURPOSE: Filter parameters for querying task logs. -# @COMPLEXITY: 3 +# @COMPLEXITY: 1 +# @RELATION: [USED_BY] -> [TaskManager] class LogFilter(BaseModel): level: Optional[str] = None # Filter by log level source: Optional[str] = None # Filter by source component search: Optional[str] = None # Text search in message offset: int = Field(default=0, ge=0) limit: int = Field(default=100, ge=1, le=1000) + + # [/DEF:LogFilter:Class] + # [DEF:LogStats:Class] # @SEMANTICS: log, stats, aggregation, pydantic # @PURPOSE: Statistics about log entries for a task. -# @COMPLEXITY: 3 +# @COMPLEXITY: 1 +# @RELATION: [COMPUTED_FROM] -> [TaskLog] class LogStats(BaseModel): total_count: int by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2} by_source: Dict[str, int] # {"plugin": 5, "superset_api": 7} + + # [/DEF:LogStats:Class] + # [DEF:Task:Class] # @COMPLEXITY: 3 # @SEMANTICS: task, job, execution, state, pydantic # @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs. +# @RELATION: [DEPENDS_ON] -> [TaskStatus] +# @RELATION: [CONTAINS] -> [LogEntry] +# @RELATION: [USED_BY] -> [TaskManager] class Task(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4())) plugin_id: str @@ -129,7 +164,10 @@ class Task(BaseModel): super().__init__(**data) if self.status == TaskStatus.AWAITING_INPUT and not self.input_request: raise ValueError("input_request is required when status is AWAITING_INPUT") + # [/DEF:__init__:Function] + + # [/DEF:Task:Class] # [/DEF:TaskManagerModels:Module] diff --git a/backend/src/core/task_manager/persistence.py b/backend/src/core/task_manager/persistence.py index b899c956..afc737ed 100644 --- a/backend/src/core/task_manager/persistence.py +++ b/backend/src/core/task_manager/persistence.py @@ -7,7 +7,7 @@ # @POST: Provides reliable storage and retrieval for task metadata and logs. # @SIDE_EFFECT: Performs database I/O on tasks.db. # @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord] -# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager] +# @RELATION: [USED_BY] ->[TaskManager] # @RELATION: [DEPENDS_ON] ->[TasksSessionLocal] # @INVARIANT: Database schema must match the TaskRecord model structure. @@ -36,7 +36,7 @@ from ..logger import logger, belief_scope # @RELATION: [DEPENDS_ON] ->[TasksSessionLocal] # @RELATION: [DEPENDS_ON] ->[TaskRecord] # @RELATION: [DEPENDS_ON] ->[Environment] -# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager] +# @RELATION: [USED_BY] ->[TaskManager] # @INVARIANT: Persistence must handle potentially missing task fields natively. # # @TEST_CONTRACT: TaskPersistenceService -> @@ -100,6 +100,7 @@ class TaskPersistenceService: # @PRE: Session is active # @POST: Returns existing environments.id or None when unresolved. # @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]] + # @RELATION: [DEPENDS_ON] ->[Environment] @staticmethod def _resolve_environment_id(session: Session, env_id: Optional[str]) -> Optional[str]: with belief_scope("_resolve_environment_id"): @@ -287,6 +288,7 @@ class TaskPersistenceService: # @POST: Specified task records deleted from database. # @PARAM: task_ids (List[str]) - List of task IDs to delete. # @SIDE_EFFECT: Deletes rows from task_records table. + # @RELATION: [DEPENDS_ON] ->[TaskRecord] def delete_tasks(self, task_ids: List[str]) -> None: if not task_ids: return @@ -313,7 +315,7 @@ class TaskPersistenceService: # @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]] # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] # @RELATION: [DEPENDS_ON] ->[TasksSessionLocal] -# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager] +# @RELATION: [USED_BY] ->[TaskManager] # @INVARIANT: Log entries are batch-inserted for performance. # # @TEST_CONTRACT: TaskLogPersistenceService -> @@ -352,6 +354,7 @@ class TaskLogPersistenceService: # @PARAM: logs (List[LogEntry]) - Log entries to insert. # @SIDE_EFFECT: Writes to task_logs table. # @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord] + # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] def add_logs(self, task_id: str, logs: List[LogEntry]) -> None: if not logs: return @@ -385,6 +388,9 @@ class TaskLogPersistenceService: # @PARAM: log_filter (LogFilter) - Filter parameters. # @RETURN: List[TaskLog] - Filtered log entries. # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]] + # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] + # @RELATION: [DEPENDS_ON] ->[LogFilter] + # @RELATION: [DEPENDS_ON] ->[TaskLog] def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]: with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"): session: Session = TasksSessionLocal() @@ -438,6 +444,8 @@ class TaskLogPersistenceService: # @PARAM: task_id (str) - The task ID. # @RETURN: LogStats - Statistics about task logs. # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats] + # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] + # @RELATION: [DEPENDS_ON] ->[LogStats] def get_log_stats(self, task_id: str) -> LogStats: with belief_scope("TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"): session: Session = TasksSessionLocal() @@ -485,6 +493,7 @@ class TaskLogPersistenceService: # @PARAM: task_id (str) - The task ID. # @RETURN: List[str] - Unique source names. # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]] + # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] def get_sources(self, task_id: str) -> List[str]: with belief_scope("TaskLogPersistenceService.get_sources", f"task_id={task_id}"): session: Session = TasksSessionLocal() @@ -505,6 +514,7 @@ class TaskLogPersistenceService: # @POST: All logs for the task are deleted. # @PARAM: task_id (str) - The task ID. # @SIDE_EFFECT: Deletes from task_logs table. + # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] def delete_logs_for_task(self, task_id: str) -> None: with belief_scope("TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"): session: Session = TasksSessionLocal() @@ -527,6 +537,7 @@ class TaskLogPersistenceService: # @POST: All logs for the tasks are deleted. # @PARAM: task_ids (List[str]) - List of task IDs. # @SIDE_EFFECT: Deletes rows from task_logs table. + # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] def delete_logs_for_tasks(self, task_ids: List[str]) -> None: if not task_ids: return diff --git a/backend/src/core/task_manager/task_logger.py b/backend/src/core/task_manager/task_logger.py index 4d242f6d..ca3f7c80 100644 --- a/backend/src/core/task_manager/task_logger.py +++ b/backend/src/core/task_manager/task_logger.py @@ -2,18 +2,23 @@ # @SEMANTICS: task, logger, context, plugin, attribution # @PURPOSE: Provides a dedicated logger for tasks with automatic source attribution. # @LAYER: Core -# @RELATION: DEPENDS_ON -> TaskManager, CALLS -> TaskManager._add_log -# @COMPLEXITY: 5 +# @RELATION: [DEPENDS_ON] -> [TaskManager] +# @RELATION: [CALLS] -> [_add_log] +# @COMPLEXITY: 2 # @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source. # [SECTION: IMPORTS] from typing import Dict, Any, Optional, Callable # [/SECTION] + # [DEF:TaskLogger:Class] # @SEMANTICS: logger, task, source, attribution # @PURPOSE: A wrapper around TaskManager._add_log that carries task_id and source context. -# @COMPLEXITY: 5 +# @COMPLEXITY: 2 +# @RELATION: [DEPENDS_ON] -> [TaskManager] +# @RELATION: [CALLS] -> [_add_log] +# @RELATION: [USED_BY] -> [TaskManager] # @INVARIANT: All log calls include the task_id and source. # @UX_STATE: Idle -> Logging -> (system records log) # @@ -33,17 +38,17 @@ from typing import Dict, Any, Optional, Callable class TaskLogger: """ A dedicated logger for tasks that automatically tags logs with source attribution. - + Usage: logger = TaskLogger(task_id="abc123", add_log_fn=task_manager._add_log, source="plugin") logger.info("Starting backup process") logger.error("Failed to connect", metadata={"error_code": 500}) - + # Create sub-logger with different source api_logger = logger.with_source("superset_api") api_logger.info("Fetching dashboards") """ - + # [DEF:__init__:Function] # @PURPOSE: Initialize the TaskLogger with task context. # @PRE: add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata). @@ -51,17 +56,13 @@ class TaskLogger: # @PARAM: task_id (str) - The ID of the task. # @PARAM: add_log_fn (Callable) - Function to add log to TaskManager. # @PARAM: source (str) - Default source for logs (default: "plugin"). - def __init__( - self, - task_id: str, - add_log_fn: Callable, - source: str = "plugin" - ): + def __init__(self, task_id: str, add_log_fn: Callable, source: str = "plugin"): self._task_id = task_id self._add_log = add_log_fn self._default_source = source + # [/DEF:__init__:Function] - + # [DEF:with_source:Function] # @PURPOSE: Create a sub-logger with a different default source. # @PRE: source is a non-empty string. @@ -71,12 +72,11 @@ class TaskLogger: def with_source(self, source: str) -> "TaskLogger": """Create a sub-logger with a different source context.""" return TaskLogger( - task_id=self._task_id, - add_log_fn=self._add_log, - source=source + task_id=self._task_id, add_log_fn=self._add_log, source=source ) + # [/DEF:with_source:Function] - + # [DEF:_log:Function] # @PURPOSE: Internal method to log a message at a given level. # @PRE: level is a valid log level string. @@ -87,11 +87,11 @@ class TaskLogger: # @PARAM: metadata (Optional[Dict]) - Additional structured data. # @UX_STATE: Logging -> (writing internal log) def _log( - self, - level: str, - message: str, + self, + level: str, + message: str, source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: """Internal logging method.""" self._add_log( @@ -99,10 +99,11 @@ class TaskLogger: level=level, message=message, source=source or self._default_source, - metadata=metadata + metadata=metadata, ) + # [/DEF:_log:Function] - + # [DEF:debug:Function] # @PURPOSE: Log a DEBUG level message. # @PRE: message is a string. @@ -111,14 +112,15 @@ class TaskLogger: # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. def debug( - self, - message: str, + self, + message: str, source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: self._log("DEBUG", message, source, metadata) + # [/DEF:debug:Function] - + # [DEF:info:Function] # @PURPOSE: Log an INFO level message. # @PRE: message is a string. @@ -127,14 +129,15 @@ class TaskLogger: # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. def info( - self, - message: str, + self, + message: str, source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: self._log("INFO", message, source, metadata) + # [/DEF:info:Function] - + # [DEF:warning:Function] # @PURPOSE: Log a WARNING level message. # @PRE: message is a string. @@ -143,14 +146,15 @@ class TaskLogger: # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. def warning( - self, - message: str, + self, + message: str, source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: self._log("WARNING", message, source, metadata) + # [/DEF:warning:Function] - + # [DEF:error:Function] # @PURPOSE: Log an ERROR level message. # @PRE: message is a string. @@ -159,14 +163,15 @@ class TaskLogger: # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. def error( - self, - message: str, + self, + message: str, source: Optional[str] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: self._log("ERROR", message, source, metadata) + # [/DEF:error:Function] - + # [DEF:progress:Function] # @PURPOSE: Log a progress update with percentage. # @PRE: percent is between 0 and 100. @@ -175,16 +180,15 @@ class TaskLogger: # @PARAM: percent (float) - Progress percentage (0-100). # @PARAM: source (Optional[str]) - Override source. def progress( - self, - message: str, - percent: float, - source: Optional[str] = None + self, message: str, percent: float, source: Optional[str] = None ) -> None: """Log a progress update with percentage.""" metadata = {"progress": min(100, max(0, percent))} self._log("INFO", message, source, metadata) + # [/DEF:progress:Function] + # [/DEF:TaskLogger:Class] # [/DEF:TaskLoggerModule:Module] diff --git a/backend/src/models/__init__.py b/backend/src/models/__init__.py index 0ba3f8d2..4aeb09b4 100644 --- a/backend/src/models/__init__.py +++ b/backend/src/models/__init__.py @@ -1,3 +1,3 @@ -# [DEF:src.models:Package] +# [DEF:ModelsPackage:Package] # @PURPOSE: Domain model package root. -# [/DEF:src.models:Package] +# [/DEF:ModelsPackage:Package] diff --git a/backend/src/models/auth.py b/backend/src/models/auth.py index 5e4484f9..ecf662b7 100644 --- a/backend/src/models/auth.py +++ b/backend/src/models/auth.py @@ -5,7 +5,7 @@ # @SEMANTICS: auth, models, user, role, permission, sqlalchemy # @PURPOSE: SQLAlchemy models for multi-user authentication and authorization. # @LAYER: Domain -# @RELATION: INHERITS_FROM -> [MappingModels:Base] +# @RELATION: INHERITS_FROM -> [Base] # # @INVARIANT: Usernames and emails must be unique. @@ -17,19 +17,22 @@ from sqlalchemy.orm import relationship from .mapping import Base # [/SECTION] + # [DEF:generate_uuid:Function] # @PURPOSE: Generates a unique UUID string. # @POST: Returns a string representation of a new UUID. -# @RELATION: DEPENDS_ON -> uuid +# @RELATION: DEPENDS_ON -> [uuid] def generate_uuid(): return str(uuid.uuid4()) + + # [/DEF:generate_uuid:Function] # [DEF:user_roles:Table] # @PURPOSE: Association table for many-to-many relationship between Users and Roles. -# @RELATION: DEPENDS_ON -> Base.metadata -# @RELATION: DEPENDS_ON -> User -# @RELATION: DEPENDS_ON -> Role +# @RELATION: DEPENDS_ON -> [Base] +# @RELATION: DEPENDS_ON -> [User] +# @RELATION: DEPENDS_ON -> [Role] user_roles = Table( "user_roles", Base.metadata, @@ -40,9 +43,9 @@ user_roles = Table( # [DEF:role_permissions:Table] # @PURPOSE: Association table for many-to-many relationship between Roles and Permissions. -# @RELATION: DEPENDS_ON -> Base.metadata -# @RELATION: DEPENDS_ON -> Role -# @RELATION: DEPENDS_ON -> Permission +# @RELATION: DEPENDS_ON -> [Base] +# @RELATION: DEPENDS_ON -> [Role] +# @RELATION: DEPENDS_ON -> [Permission] role_permissions = Table( "role_permissions", Base.metadata, @@ -51,65 +54,81 @@ role_permissions = Table( ) # [/DEF:role_permissions:Table] + # [DEF:User:Class] # @PURPOSE: Represents an identity that can authenticate to the system. -# @RELATION: HAS_MANY -> Role (via user_roles) +# @RELATION: HAS_MANY -> [Role] class User(Base): __tablename__ = "users" - + id = Column(String, primary_key=True, default=generate_uuid) username = Column(String, unique=True, index=True, nullable=False) email = Column(String, unique=True, index=True, nullable=True) password_hash = Column(String, nullable=True) full_name = Column(String, nullable=True) - auth_source = Column(String, default="LOCAL") # LOCAL or ADFS + auth_source = Column(String, default="LOCAL") # LOCAL or ADFS is_active = Column(Boolean, default=True) is_ad_user = Column(Boolean, default=False) created_at = Column(DateTime, default=datetime.utcnow) last_login = Column(DateTime, nullable=True) - + roles = relationship("Role", secondary=user_roles, back_populates="users") + + # [/DEF:User:Class] + # [DEF:Role:Class] # @PURPOSE: Represents a collection of permissions. -# @RELATION: HAS_MANY -> User (via user_roles) -# @RELATION: HAS_MANY -> Permission (via role_permissions) +# @RELATION: HAS_MANY -> [User] +# @RELATION: HAS_MANY -> [Permission] class Role(Base): __tablename__ = "roles" - + id = Column(String, primary_key=True, default=generate_uuid) name = Column(String, unique=True, index=True, nullable=False) description = Column(String, nullable=True) - + users = relationship("User", secondary=user_roles, back_populates="roles") - permissions = relationship("Permission", secondary=role_permissions, back_populates="roles") + permissions = relationship( + "Permission", secondary=role_permissions, back_populates="roles" + ) + + # [/DEF:Role:Class] + # [DEF:Permission:Class] # @PURPOSE: Represents a specific capability within the system. -# @RELATION: HAS_MANY -> Role (via role_permissions) +# @RELATION: HAS_MANY -> [Role] class Permission(Base): __tablename__ = "permissions" - + id = Column(String, primary_key=True, default=generate_uuid) - resource = Column(String, nullable=False) # e.g. "plugin:backup" - action = Column(String, nullable=False) # e.g. "READ", "EXECUTE", "WRITE" - - roles = relationship("Role", secondary=role_permissions, back_populates="permissions") + resource = Column(String, nullable=False) # e.g. "plugin:backup" + action = Column(String, nullable=False) # e.g. "READ", "EXECUTE", "WRITE" + + roles = relationship( + "Role", secondary=role_permissions, back_populates="permissions" + ) + + # [/DEF:Permission:Class] + # [DEF:ADGroupMapping:Class] # @PURPOSE: Maps an Active Directory group to a local System Role. -# @RELATION: DEPENDS_ON -> Role +# @RELATION: DEPENDS_ON -> [Role] class ADGroupMapping(Base): __tablename__ = "ad_group_mappings" - + id = Column(String, primary_key=True, default=generate_uuid) ad_group = Column(String, unique=True, index=True, nullable=False) role_id = Column(String, ForeignKey("roles.id"), nullable=False) - + role = relationship("Role") + + # [/DEF:ADGroupMapping:Class] -# [/DEF:AuthModels:Module] \ No newline at end of file +# [/DEF:AuthModels:Module] diff --git a/backend/src/scripts/init_auth_db.py b/backend/src/scripts/init_auth_db.py index b65f7b4d..83627c5a 100644 --- a/backend/src/scripts/init_auth_db.py +++ b/backend/src/scripts/init_auth_db.py @@ -1,9 +1,12 @@ -# [DEF:backend.src.scripts.init_auth_db:Module] +# [DEF:InitAuthDbScript:Module] # # @SEMANTICS: setup, database, auth, migration # @PURPOSE: Initializes the auth database and creates the necessary tables. +# @COMPLEXITY: 2 # @LAYER: Scripts -# @RELATION: CALLS -> backend.src.core.database.init_db +# @RELATION: CALLS -> init_db +# @RELATION: CALLS -> ensure_encryption_key +# @RELATION: CALLS -> seed_permissions # # @INVARIANT: Safe to run multiple times (idempotent). @@ -20,9 +23,14 @@ from src.core.logger import logger, belief_scope from src.scripts.seed_permissions import seed_permissions # [/SECTION] + # [DEF:run_init:Function] # @PURPOSE: Main entry point for the initialization script. +# @COMPLEXITY: 3 # @POST: auth.db is initialized with the correct schema and seeded permissions. +# @RELATION: CALLS -> ensure_encryption_key +# @RELATION: CALLS -> init_db +# @RELATION: CALLS -> seed_permissions def run_init(): with belief_scope("init_auth_db"): logger.info("Initializing authentication database...") @@ -30,16 +38,18 @@ def run_init(): ensure_encryption_key() init_db() logger.info("Authentication database initialized successfully.") - + # Seed permissions seed_permissions() - + except Exception as e: logger.error(f"Failed to initialize authentication database: {e}") sys.exit(1) + + # [/DEF:run_init:Function] if __name__ == "__main__": run_init() -# [/DEF:backend.src.scripts.init_auth_db:Module] +# [/DEF:InitAuthDbScript:Module] diff --git a/backend/src/scripts/seed_permissions.py b/backend/src/scripts/seed_permissions.py index 67651f27..0ba255dd 100644 --- a/backend/src/scripts/seed_permissions.py +++ b/backend/src/scripts/seed_permissions.py @@ -1,10 +1,13 @@ -# [DEF:backend.src.scripts.seed_permissions:Module] +# [DEF:SeedPermissionsScript:Module] # # @SEMANTICS: setup, database, auth, permissions, seeding # @PURPOSE: Populates the auth database with initial system permissions. +# @COMPLEXITY: 3 # @LAYER: Scripts -# @RELATION: USES -> backend.src.core.database.get_auth_db -# @RELATION: USES -> backend.src.models.auth.Permission +# @RELATION: DEPENDS_ON -> AuthSessionLocal +# @RELATION: DEPENDS_ON -> Permission +# @RELATION: DEPENDS_ON -> Role +# @RELATION: DEPENDS_ON -> AuthRepository # # @INVARIANT: Safe to run multiple times (idempotent). @@ -22,6 +25,9 @@ from src.core.logger import logger, belief_scope # [/SECTION] # [DEF:INITIAL_PERMISSIONS:Constant] +# @PURPOSE: Canonical bootstrap permission tuples seeded into auth storage. +# @COMPLEXITY: 3 +# @RELATION: DEPENDS_ON -> SeedPermissionsScript INITIAL_PERMISSIONS = [ # Admin Permissions {"resource": "admin:users", "action": "READ"}, @@ -34,7 +40,6 @@ INITIAL_PERMISSIONS = [ {"resource": "plugins", "action": "READ"}, {"resource": "tasks", "action": "READ"}, {"resource": "tasks", "action": "WRITE"}, - # Plugin Permissions {"resource": "plugin:backup", "action": "EXECUTE"}, {"resource": "plugin:migration", "action": "EXECUTE"}, @@ -46,7 +51,6 @@ INITIAL_PERMISSIONS = [ {"resource": "plugin:storage", "action": "WRITE"}, {"resource": "plugin:debug", "action": "EXECUTE"}, {"resource": "git_config", "action": "READ"}, - # Dataset Review Permissions {"resource": "dataset:session", "action": "READ"}, {"resource": "dataset:session", "action": "MANAGE"}, @@ -57,9 +61,16 @@ INITIAL_PERMISSIONS = [ ] # [/DEF:INITIAL_PERMISSIONS:Constant] + # [DEF:seed_permissions:Function] # @PURPOSE: Inserts missing permissions into the database. +# @COMPLEXITY: 3 # @POST: All INITIAL_PERMISSIONS exist in the DB. +# @RELATION: DEPENDS_ON -> AuthSessionLocal +# @RELATION: DEPENDS_ON -> Permission +# @RELATION: DEPENDS_ON -> Role +# @RELATION: DEPENDS_ON -> AuthRepository +# @RELATION: DEPENDS_ON -> INITIAL_PERMISSIONS def seed_permissions(): with belief_scope("seed_permissions"): db = AuthSessionLocal() @@ -67,19 +78,22 @@ def seed_permissions(): logger.info("Seeding permissions...") count = 0 for perm_data in INITIAL_PERMISSIONS: - exists = db.query(Permission).filter( - Permission.resource == perm_data["resource"], - Permission.action == perm_data["action"] - ).first() - + exists = ( + db.query(Permission) + .filter( + Permission.resource == perm_data["resource"], + Permission.action == perm_data["action"], + ) + .first() + ) + if not exists: new_perm = Permission( - resource=perm_data["resource"], - action=perm_data["action"] + resource=perm_data["resource"], action=perm_data["action"] ) db.add(new_perm) count += 1 - + db.commit() logger.info(f"Seeding completed. Added {count} new permissions.") @@ -87,10 +101,12 @@ def seed_permissions(): repo = AuthRepository(db) user_role = repo.get_role_by_name("User") if not user_role: - user_role = Role(name="User", description="Standard user with plugin access") + user_role = Role( + name="User", description="Standard user with plugin access" + ) db.add(user_role) db.flush() - + user_permissions = [ ("plugin:mapper", "EXECUTE"), ("plugin:migration", "EXECUTE"), @@ -113,7 +129,7 @@ def seed_permissions(): perm = repo.get_permission_by_resource_action(res, act) if perm and perm not in user_role.permissions: user_role.permissions.append(perm) - + db.commit() logger.info("User role permissions updated.") @@ -122,9 +138,11 @@ def seed_permissions(): db.rollback() finally: db.close() + + # [/DEF:seed_permissions:Function] if __name__ == "__main__": seed_permissions() -# [/DEF:backend.src.scripts.seed_permissions:Module] \ No newline at end of file +# [/DEF:SeedPermissionsScript:Module] diff --git a/backend/src/scripts/test_dataset_dashboard_relations.py b/backend/src/scripts/test_dataset_dashboard_relations.py index 8987342c..96844e6b 100644 --- a/backend/src/scripts/test_dataset_dashboard_relations.py +++ b/backend/src/scripts/test_dataset_dashboard_relations.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# [DEF:test_dataset_dashboard_relations_script:Module] +# @SEMANTICS: scripts, test, dataset, dashboard, superset, relations +# @PURPOSE: Tests and inspects dataset-to-dashboard relationship responses from Superset API. +# @COMPLEXITY: 2 """ Script to test dataset-to-dashboard relationships from Superset API. @@ -164,3 +168,5 @@ def test_dashboard_dataset_relations(): if __name__ == "__main__": test_dashboard_dataset_relations() + +# [/DEF:test_dataset_dashboard_relations_script:Module] diff --git a/backend/src/services/__tests__/test_llm_plugin_persistence.py b/backend/src/services/__tests__/test_llm_plugin_persistence.py index 5bfeef8a..3be0ac3b 100644 --- a/backend/src/services/__tests__/test_llm_plugin_persistence.py +++ b/backend/src/services/__tests__/test_llm_plugin_persistence.py @@ -1,5 +1,5 @@ # [DEF:test_llm_plugin_persistence:Module] -# @RELATION: VERIFIES ->[src.plugins.llm_analysis.plugin.DashboardValidationPlugin] +# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class] # @COMPLEXITY: 3 # @PURPOSE: Regression test for ValidationRecord persistence fields populated from task context. @@ -10,7 +10,7 @@ from src.plugins.llm_analysis import plugin as plugin_module # [DEF:_DummyLogger:Class] -# @RELATION: BINDS_TO ->[test_llm_plugin_persistence] +# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] # @COMPLEXITY: 1 # @PURPOSE: Minimal logger shim for TaskContext-like objects used in tests. # @INVARIANT: Logging methods are no-ops and must not mutate test state. @@ -35,7 +35,7 @@ class _DummyLogger: # [DEF:_FakeDBSession:Class] -# @RELATION: BINDS_TO ->[test_llm_plugin_persistence] +# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] # @COMPLEXITY: 2 # @PURPOSE: Captures persisted records for assertion and mimics SQLAlchemy session methods used by plugin. # @INVARIANT: add/commit/close provide only persistence signals asserted by this test. @@ -59,8 +59,11 @@ class _FakeDBSession: # [DEF:test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] -# @RELATION: BINDS_TO ->[test_llm_plugin_persistence] +# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] +# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class] +# @COMPLEXITY: 2 # @PURPOSE: Ensure db ValidationRecord includes context.task_id and params.environment_id. +# @INVARIANT: Assertions remain restricted to persisted task/environment identity fields and session lifecycle signals. @pytest.mark.asyncio async def test_dashboard_validation_plugin_persists_task_and_environment_ids( tmp_path, monkeypatch @@ -77,6 +80,11 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( is_active=True, ) + # [DEF:_FakeProviderService:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: LLM provider service stub returning deterministic provider and decrypted API key for plugin tests. + # @INVARIANT: Returns same provider and key regardless of provider_id argument; no lookup logic. class _FakeProviderService: def __init__(self, _db): return None @@ -87,6 +95,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( def get_decrypted_api_key(self, _provider_id): return "a" * 32 + # [/DEF:_FakeProviderService:Class] + + # [DEF:_FakeScreenshotService:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Screenshot service stub that accepts capture_dashboard calls without side effects. + # @INVARIANT: capture_dashboard is intentionally permissive for this persistence-focused test and does not validate argument values. class _FakeScreenshotService: def __init__(self, _env): return None @@ -94,8 +109,10 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( async def capture_dashboard(self, _dashboard_id, _screenshot_path): return None + # [/DEF:_FakeScreenshotService:Class] + # [DEF:_FakeLLMClient:Class] - # @RELATION: BINDS_TO ->[test_dashboard_validation_plugin_persists_task_and_environment_ids] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] # @COMPLEXITY: 2 # @PURPOSE: Deterministic LLM client double returning canonical analysis payload for persistence-path assertions. # @INVARIANT: analyze_dashboard is side-effect free and returns schema-compatible PASS result. @@ -112,6 +129,11 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( # [/DEF:_FakeLLMClient:Class] + # [DEF:_FakeNotificationService:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Notification service stub that accepts plugin dispatch_report payload without introducing side effects. + # @INVARIANT: dispatch_report accepts arbitrary keyword payloads because this test verifies persistence fields, not notification payload schema. class _FakeNotificationService: def __init__(self, *_args, **_kwargs): return None @@ -119,6 +141,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( async def dispatch_report(self, **_kwargs): return None + # [/DEF:_FakeNotificationService:Class] + + # [DEF:_FakeConfigManager:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Config manager stub providing storage root path and minimal settings for plugin execution path. + # @INVARIANT: Only storage.root_path and llm fields are safe to access; all other settings fields are absent. class _FakeConfigManager: def get_environment(self, _env_id): return env @@ -131,12 +160,21 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( ) ) + # [/DEF:_FakeConfigManager:Class] + + # [DEF:_FakeSupersetClient:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Superset client stub exposing network.request as a lambda that returns empty result list. + # @INVARIANT: network.request intentionally accepts arbitrary keyword payloads because response shape, not request signature, is the persistence-path dependency. class _FakeSupersetClient: def __init__(self, _env): self.network = types.SimpleNamespace( request=lambda **_kwargs: {"result": []} ) + # [/DEF:_FakeSupersetClient:Class] + monkeypatch.setattr(plugin_module, "SessionLocal", lambda: fake_db) monkeypatch.setattr(plugin_module, "LLMProviderService", _FakeProviderService) monkeypatch.setattr(plugin_module, "ScreenshotService", _FakeScreenshotService) diff --git a/backend/src/services/__tests__/test_llm_prompt_templates.py b/backend/src/services/__tests__/test_llm_prompt_templates.py index c2bdf586..0ca7b141 100644 --- a/backend/src/services/__tests__/test_llm_prompt_templates.py +++ b/backend/src/services/__tests__/test_llm_prompt_templates.py @@ -3,7 +3,7 @@ # @SEMANTICS: tests, llm, prompts, templates, settings # @PURPOSE: Validate normalization and rendering behavior for configurable LLM prompt templates. # @LAYER: Domain Tests -# @RELATION: DEPENDS_ON ->[backend.src.services.llm_prompt_templates:Function] +# @RELATION: DEPENDS_ON -> [backend.src.services.llm_prompt_templates:Function] # @INVARIANT: All required prompt keys remain available after normalization. from src.services.llm_prompt_templates import ( @@ -23,8 +23,6 @@ from src.services.llm_prompt_templates import ( # @PURPOSE: Ensure legacy/partial llm settings are expanded with all prompt defaults. # @PRE: Input llm settings do not contain complete prompts object. # @POST: Returned structure includes required prompt templates with fallback defaults. -# [DEF:test_normalize_llm_settings_adds_default_prompts:Function] -# @RELATION: BINDS_TO -> test_llm_prompt_templates def test_normalize_llm_settings_adds_default_prompts(): normalized = normalize_llm_settings({"default_provider": "x"}) @@ -38,6 +36,8 @@ def test_normalize_llm_settings_adds_default_prompts(): assert key in normalized["provider_bindings"] for key in DEFAULT_LLM_ASSISTANT_SETTINGS: assert key in normalized + + # [/DEF:test_normalize_llm_settings_adds_default_prompts:Function] @@ -47,17 +47,13 @@ def test_normalize_llm_settings_adds_default_prompts(): # @PURPOSE: Ensure user-customized prompt values are preserved during normalization. # @PRE: Input llm settings contain custom prompt override. # @POST: Custom prompt value remains unchanged in normalized output. -# [/DEF:test_normalize_llm_settings_adds_default_prompts:Function] - -# [DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function] -# @RELATION: BINDS_TO -> test_llm_prompt_templates def test_normalize_llm_settings_keeps_custom_prompt_values(): custom = "Doc for {dataset_name} using {columns_json}" - normalized = normalize_llm_settings( - {"prompts": {"documentation_prompt": custom}} - ) + normalized = normalize_llm_settings({"prompts": {"documentation_prompt": custom}}) assert normalized["prompts"]["documentation_prompt"] == custom + + # [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function] @@ -67,10 +63,6 @@ def test_normalize_llm_settings_keeps_custom_prompt_values(): # @PURPOSE: Ensure template placeholders are deterministically replaced. # @PRE: Template contains placeholders matching provided variables. # @POST: Rendered prompt string contains substituted values. -# [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function] - -# [DEF:test_render_prompt_replaces_known_placeholders:Function] -# @RELATION: BINDS_TO -> test_llm_prompt_templates def test_render_prompt_replaces_known_placeholders(): rendered = render_prompt( "Hello {name}, diff={diff}", @@ -78,6 +70,8 @@ def test_render_prompt_replaces_known_placeholders(): ) assert rendered == "Hello bot, diff=A->B" + + # [/DEF:test_render_prompt_replaces_known_placeholders:Function] @@ -85,13 +79,13 @@ def test_render_prompt_replaces_known_placeholders(): # @RELATION: BINDS_TO -> test_llm_prompt_templates # @COMPLEXITY: 2 # @PURPOSE: Ensure multimodal model detection recognizes common vision-capable model names. -# [/DEF:test_render_prompt_replaces_known_placeholders:Function] - def test_is_multimodal_model_detects_known_vision_models(): assert is_multimodal_model("gpt-4o") is True assert is_multimodal_model("claude-3-5-sonnet") is True assert is_multimodal_model("stepfun/step-3.5-flash:free", "openrouter") is False assert is_multimodal_model("text-only-model") is False + + # [/DEF:test_is_multimodal_model_detects_known_vision_models:Function] @@ -106,6 +100,8 @@ def test_resolve_bound_provider_id_prefers_binding_then_default(): } assert resolve_bound_provider_id(settings, "dashboard_validation") == "vision-1" assert resolve_bound_provider_id(settings, "documentation") == "default-1" + + # [/DEF:test_resolve_bound_provider_id_prefers_binding_then_default:Function] @@ -122,6 +118,8 @@ def test_normalize_llm_settings_keeps_assistant_planner_settings(): ) assert normalized["assistant_planner_provider"] == "provider-a" assert normalized["assistant_planner_model"] == "gpt-4.1-mini" + + # [/DEF:test_normalize_llm_settings_keeps_assistant_planner_settings:Function] diff --git a/backend/src/services/__tests__/test_llm_provider.py b/backend/src/services/__tests__/test_llm_provider.py index c7b811d4..98d23133 100644 --- a/backend/src/services/__tests__/test_llm_provider.py +++ b/backend/src/services/__tests__/test_llm_provider.py @@ -1,7 +1,9 @@ -# [DEF:__tests__/test_llm_provider:Module] -# @RELATION: VERIFIES -> ../llm_provider.py +# [DEF:test_llm_provider:Module] +# @RELATION: VERIFIES -> [src.services.llm_provider:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, llm-provider, encryption, contract # @PURPOSE: Contract testing for LLMProviderService and EncryptionManager -# [/DEF:__tests__/test_llm_provider:Module] +# [/DEF:test_llm_provider:Module] import pytest import os @@ -14,14 +16,16 @@ from src.plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType # [DEF:_test_encryption_key_fixture:Global] # @PURPOSE: Ensure encryption-dependent provider tests run with a valid Fernet key. -# @RELATION: DEPENDS_ON ->[pytest:Module] +# @RELATION: DEPENDS_ON -> [pytest:Module] os.environ.setdefault("ENCRYPTION_KEY", Fernet.generate_key().decode()) # [/DEF:_test_encryption_key_fixture:Global] + # @TEST_CONTRACT: EncryptionManagerModel -> Invariants # @TEST_INVARIANT: symmetric_encryption # [DEF:test_encryption_cycle:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify EncryptionManager round-trip encryption/decryption invariant for non-empty secrets. def test_encryption_cycle(): """Verify encrypted data can be decrypted back to original string.""" manager = EncryptionManager() @@ -30,49 +34,78 @@ def test_encryption_cycle(): assert encrypted != original assert manager.decrypt(encrypted) == original + # @TEST_EDGE: empty_string_encryption # [/DEF:test_encryption_cycle:Function] + # [DEF:test_empty_string_encryption:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify EncryptionManager preserves empty-string payloads through encrypt/decrypt cycle. def test_empty_string_encryption(): manager = EncryptionManager() original = "" encrypted = manager.encrypt(original) assert manager.decrypt(encrypted) == "" + # @TEST_EDGE: decrypt_invalid_data # [/DEF:test_empty_string_encryption:Function] + # [DEF:test_decrypt_invalid_data:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Ensure decrypt rejects invalid ciphertext input by raising an exception. def test_decrypt_invalid_data(): manager = EncryptionManager() with pytest.raises(Exception): manager.decrypt("not-encrypted-string") + # @TEST_FIXTURE: mock_db_session # [/DEF:test_decrypt_invalid_data:Function] + +# [DEF:mock_db:Fixture] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @COMPLEXITY: 1 +# @PURPOSE: MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests. +# @INVARIANT: Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced. @pytest.fixture def mock_db(): return MagicMock(spec=Session) + +# [/DEF:mock_db:Fixture] + + +# [DEF:service:Fixture] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @COMPLEXITY: 1 +# @PURPOSE: LLMProviderService fixture wired to mock_db for provider CRUD tests. @pytest.fixture def service(mock_db): return LLMProviderService(db=mock_db) + +# [/DEF:service:Fixture] + + # [DEF:test_get_all_providers:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify provider list retrieval issues query/all calls on the backing DB session. def test_get_all_providers(service, mock_db): service.get_all_providers() mock_db.query.assert_called() mock_db.query().all.assert_called() + # [/DEF:test_get_all_providers:Function] + # [DEF:test_create_provider:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Ensure provider creation persists entity and stores API key in encrypted form. def test_create_provider(service, mock_db): config = LLMProviderConfig( provider_type=LLMProviderType.OPENAI, @@ -80,11 +113,11 @@ def test_create_provider(service, mock_db): base_url="https://api.openai.com", api_key="sk-test", default_model="gpt-4", - is_active=True + is_active=True, ) - + provider = service.create_provider(config) - + mock_db.add.assert_called() mock_db.commit.assert_called() # Verify API key was encrypted @@ -92,31 +125,40 @@ def test_create_provider(service, mock_db): # Decrypt to verify it matches assert EncryptionManager().decrypt(provider.api_key) == "sk-test" + # [/DEF:test_create_provider:Function] + # [DEF:test_get_decrypted_api_key:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify service decrypts stored provider API key for an existing provider record. def test_get_decrypted_api_key(service, mock_db): # Setup mock provider encrypted_key = EncryptionManager().encrypt("secret-value") mock_provider = LLMProvider(id="p1", api_key=encrypted_key) mock_db.query().filter().first.return_value = mock_provider - + key = service.get_decrypted_api_key("p1") assert key == "secret-value" + # [/DEF:test_get_decrypted_api_key:Function] + # [DEF:test_get_decrypted_api_key_not_found:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify missing provider lookup returns None instead of attempting decryption. def test_get_decrypted_api_key_not_found(service, mock_db): mock_db.query().filter().first.return_value = None assert service.get_decrypted_api_key("missing") is None + # [/DEF:test_get_decrypted_api_key_not_found:Function] + # [DEF:test_update_provider_ignores_masked_placeholder_api_key:Function] -# @RELATION: BINDS_TO -> __tests__/test_llm_provider +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Ensure masked placeholder API keys do not overwrite previously encrypted provider secrets. def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db): existing_encrypted = EncryptionManager().encrypt("secret-value") mock_provider = LLMProvider( @@ -145,4 +187,6 @@ def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db): assert updated.api_key == existing_encrypted assert EncryptionManager().decrypt(updated.api_key) == "secret-value" assert updated.is_active is False + + # [/DEF:test_update_provider_ignores_masked_placeholder_api_key:Function] diff --git a/backend/src/services/auth_service.py b/backend/src/services/auth_service.py index 06c00143..f41367b5 100644 --- a/backend/src/services/auth_service.py +++ b/backend/src/services/auth_service.py @@ -3,11 +3,11 @@ # @SEMANTICS: auth, service, business-logic, login, jwt, adfs, jit-provisioning # @PURPOSE: Orchestrates credential authentication and ADFS JIT user provisioning. # @LAYER: Domain -# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.repository.AuthRepository] -# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.security.verify_password] -# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.jwt.create_access_token] -# @RELATION: [DEPENDS_ON] ->[backend.src.models.auth.User] -# @RELATION: [DEPENDS_ON] ->[backend.src.models.auth.Role] +# @RELATION: DEPENDS_ON -> [AuthRepository] +# @RELATION: DEPENDS_ON -> [verify_password] +# @RELATION: DEPENDS_ON -> [create_access_token] +# @RELATION: DEPENDS_ON -> [User] +# @RELATION: DEPENDS_ON -> [Role] # @INVARIANT: Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles. # @PRE: Core auth models and security utilities available. # @POST: User identity verified and session tokens issued according to role scopes. @@ -41,9 +41,10 @@ class AuthService: def __init__(self, db: Session): self.db = db self.repo = AuthRepository(db) + # [/DEF:AuthService_init:Function] - # [DEF:authenticate_user:Function] + # [DEF:AuthService.authenticate_user:Function] # @COMPLEXITY: 3 # @PURPOSE: Validates credentials and account state for local username/password authentication. # @PRE: username and password are non-empty credential inputs. @@ -58,19 +59,20 @@ class AuthService: user = self.repo.get_user_by_username(username) if not user or not user.is_active: return None - + if not verify_password(password, user.password_hash): return None - + # Update last login user.last_login = datetime.utcnow() self.db.commit() self.db.refresh(user) - - return user - # [/DEF:authenticate_user:Function] - # [DEF:create_session:Function] + return user + + # [/DEF:AuthService.authenticate_user:Function] + + # [DEF:AuthService.create_session:Function] # @COMPLEXITY: 3 # @PURPOSE: Issues an access token payload for an already authenticated user. # @PRE: user is a valid User entity containing username and iterable roles with role.name values. @@ -86,9 +88,10 @@ class AuthService: data={"sub": user.username, "scopes": roles} ) return {"access_token": access_token, "token_type": "bearer"} - # [/DEF:create_session:Function] - # [DEF:provision_adfs_user:Function] + # [/DEF:AuthService.create_session:Function] + + # [DEF:AuthService.provision_adfs_user:Function] # @COMPLEXITY: 3 # @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings. # @PRE: user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent. @@ -102,7 +105,7 @@ class AuthService: username = user_info.get("upn") or user_info.get("email") email = user_info.get("email") groups = user_info.get("groups", []) - + user = self.repo.get_user_by_username(username) if not user: user = User( @@ -111,21 +114,24 @@ class AuthService: full_name=user_info.get("name"), auth_source="ADFS", is_active=True, - is_ad_user=True + is_ad_user=True, ) self.db.add(user) log_security_event("USER_PROVISIONED", username, {"source": "ADFS"}) - + # Sync roles from AD groups mapped_roles = self.repo.get_roles_by_ad_groups(groups) user.roles = mapped_roles - + user.last_login = datetime.utcnow() self.db.commit() self.db.refresh(user) - + return user - # [/DEF:provision_adfs_user:Function] + + # [/DEF:AuthService.provision_adfs_user:Function] + + # [/DEF:AuthService:Class] -# [/DEF:auth_service:Module] \ No newline at end of file +# [/DEF:auth_service:Module] diff --git a/backend/src/services/dataset_review/orchestrator.py b/backend/src/services/dataset_review/orchestrator.py index fa5598df..6e5d17f4 100644 --- a/backend/src/services/dataset_review/orchestrator.py +++ b/backend/src/services/dataset_review/orchestrator.py @@ -5,10 +5,10 @@ # @LAYER: Domain # @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository] # @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver] -# @RELATION: [DEPENDS_ON] ->[ClarificationEngine] # @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor] # @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter] # @RELATION: [DEPENDS_ON] ->[TaskManager] +# @RELATION: [CONTAINS] ->[DatasetReviewOrchestrator] # @PRE: session mutations must execute inside a persisted session boundary scoped to one authenticated user. # @POST: state transitions are persisted atomically and emit observable progress for long-running steps. # @SIDE_EFFECT: creates task records, updates session aggregates, triggers upstream Superset calls, persists audit artifacts. @@ -158,8 +158,8 @@ class LaunchDatasetResult: # @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository] # @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor] # @RELATION: [DEPENDS_ON] ->[TaskManager] -# @RELATION: [DEPENDS_ON] ->[SessionRepo] # @RELATION: [DEPENDS_ON] ->[ConfigManager] +# @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver] # @PRE: constructor dependencies are valid and tied to the current request/task scope. # @POST: orchestrator instance can execute session-scoped mutations for one authenticated user. # @SIDE_EFFECT: downstream operations may persist session/profile/finding state and enqueue background tasks. @@ -169,8 +169,13 @@ class DatasetReviewOrchestrator: # [DEF:DatasetReviewOrchestrator_init:Function] # @COMPLEXITY: 3 # @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary. - # @RELATION: [DEPENDS_ON] ->[SessionRepo] + # @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository] # @RELATION: [DEPENDS_ON] ->[ConfigManager] + # @RELATION: [DEPENDS_ON] ->[TaskManager] + # @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver] + # @PRE: repository/config_manager are valid collaborators for the current request scope. + # @POST: Instance holds collaborator references used by start/preview/launch orchestration methods. + # @SIDE_EFFECT: Stores dependency references for later session lifecycle operations. def __init__( self, repository: DatasetReviewSessionRepository, @@ -188,9 +193,9 @@ class DatasetReviewOrchestrator: # [DEF:start_session:Function] # @COMPLEXITY: 5 # @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery. - # @RELATION: [DEPENDS_ON] ->[SessionRepo] + # @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository] # @RELATION: [CALLS] ->[SupersetContextExtractor.parse_superset_link] - # @RELATION: [CALLS] ->[create_task] + # @RELATION: [CALLS] ->[TaskManager.create_task] # @PRE: source input is non-empty and environment is accessible. # @POST: session exists in persisted storage with intake/recovery state and task linkage when async work is required. # @SIDE_EFFECT: persists session and may enqueue recovery task. @@ -1112,7 +1117,7 @@ class DatasetReviewOrchestrator: # [DEF:_enqueue_recovery_task:Function] # @COMPLEXITY: 4 # @PURPOSE: Link session start to observable async recovery when task infrastructure is available. - # @RELATION: [CALLS] ->[create_task] + # @RELATION: [CALLS] ->[TaskManager.create_task] # @PRE: session is already persisted. # @POST: returns task identifier when a task could be enqueued, otherwise None. # @SIDE_EFFECT: may create one background task for progressive recovery. diff --git a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py index c6f29ddc..2c4d35be 100644 --- a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py +++ b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py @@ -16,34 +16,46 @@ from src.models.dataset_review import ( ReadinessState, RecommendedAction, SessionCollaborator, - SessionCollaboratorRole + SessionCollaboratorRole, +) +from src.services.dataset_review.repositories.session_repository import ( + DatasetReviewSessionRepository, ) -from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository # [DEF:SessionRepositoryTests:Module] # @RELATION: BELONGS_TO -> SrcRoot # @COMPLEXITY: 2 # @PURPOSE: Unit tests for DatasetReviewSessionRepository. + @pytest.fixture def db_session(): # [DEF:db_session:Function] # @COMPLEXITY: 2 + # @PURPOSE: Build isolated in-memory SQLAlchemy session seeded with baseline user/environment rows. # @RELATION: BINDS_TO -> [SessionRepositoryTests] engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(engine) Session = sessionmaker(bind=engine) session = Session() - + # Create test data - user = User(id="user1", username="testuser", email="test@example.com", password_hash="pw") - env = Environment(id="env1", name="Prod", url="http://superset", credentials_id="cred1") + user = User( + id="user1", username="testuser", email="test@example.com", password_hash="pw" + ) + env = Environment( + id="env1", name="Prod", url="http://superset", credentials_id="cred1" + ) session.add_all([user, env]) session.commit() - + yield session session.close() + +# [/DEF:db_session:Function] + + # [DEF:test_create_session:Function] # @RELATION: BINDS_TO -> SessionRepositoryTests def test_create_session(db_session): @@ -54,126 +66,156 @@ def test_create_session(db_session): environment_id="env1", source_kind="superset_link", source_input="http://link", - dataset_ref="dataset1" + dataset_ref="dataset1", ) repo.create_session(session) - + assert session.session_id is not None - loaded = db_session.query(DatasetReviewSession).filter_by(session_id=session.session_id).first() + loaded = ( + db_session.query(DatasetReviewSession) + .filter_by(session_id=session.session_id) + .first() + ) assert loaded.user_id == "user1" + # [/DEF:test_create_session:Function] + # [DEF:test_load_session_detail_ownership:Function] # @RELATION: BINDS_TO -> SessionRepositoryTests def test_load_session_detail_ownership(db_session): # @PURPOSE: Verify ownership enforcement in detail loading. repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( - user_id="user1", environment_id="env1", source_kind="superset_link", - source_input="http://link", dataset_ref="dataset1" + user_id="user1", + environment_id="env1", + source_kind="superset_link", + source_input="http://link", + dataset_ref="dataset1", ) repo.create_session(session) - + # Correct user loaded = repo.load_session_detail(session.session_id, "user1") assert loaded is not None - + # Wrong user loaded_wrong = repo.load_session_detail(session.session_id, "wrong_user") assert loaded_wrong is None + # [/DEF:test_load_session_detail_ownership:Function] + # [DEF:test_load_session_detail_collaborator:Function] # @RELATION: BINDS_TO -> SessionRepositoryTests def test_load_session_detail_collaborator(db_session): # @PURPOSE: Verify collaborator access in detail loading. repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( - user_id="user1", environment_id="env1", source_kind="superset_link", - source_input="http://link", dataset_ref="dataset1" + user_id="user1", + environment_id="env1", + source_kind="superset_link", + source_input="http://link", + dataset_ref="dataset1", ) repo.create_session(session) - + # Add collaborator - collab_user = User(id="collab1", username="collab", email="c@e.com", password_hash="p") + collab_user = User( + id="collab1", username="collab", email="c@e.com", password_hash="p" + ) db_session.add(collab_user) - + collaborator = SessionCollaborator( session_id=session.session_id, user_id="collab1", - role=SessionCollaboratorRole.REVIEWER + role=SessionCollaboratorRole.REVIEWER, ) db_session.add(collaborator) db_session.commit() - + # Collaborator access loaded = repo.load_session_detail(session.session_id, "collab1") assert loaded is not None assert loaded.session_id == session.session_id + # [/DEF:test_load_session_detail_collaborator:Function] + # [DEF:test_save_preview_marks_stale:Function] # @RELATION: BINDS_TO -> SessionRepositoryTests def test_save_preview_marks_stale(db_session): # @PURPOSE: Verify that saving a new preview marks old ones as stale. repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( - user_id="user1", environment_id="env1", source_kind="superset_link", - source_input="http://link", dataset_ref="dataset1" + user_id="user1", + environment_id="env1", + source_kind="superset_link", + source_input="http://link", + dataset_ref="dataset1", ) repo.create_session(session) - - p1 = CompiledPreview(session_id=session.session_id, preview_status="ready", preview_fingerprint="f1") + + p1 = CompiledPreview( + session_id=session.session_id, preview_status="ready", preview_fingerprint="f1" + ) repo.save_preview(session.session_id, "user1", p1) - - p2 = CompiledPreview(session_id=session.session_id, preview_status="ready", preview_fingerprint="f2") + + p2 = CompiledPreview( + session_id=session.session_id, preview_status="ready", preview_fingerprint="f2" + ) repo.save_preview(session.session_id, "user1", p2) - + db_session.refresh(p1) assert p1.preview_status == "stale" assert p2.preview_status == "ready" assert session.last_preview_id == p2.preview_id + # [/DEF:test_save_preview_marks_stale:Function] + # [DEF:test_save_profile_and_findings:Function] # @RELATION: BINDS_TO -> SessionRepositoryTests def test_save_profile_and_findings(db_session): # @PURPOSE: Verify persistence of profile and findings. repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( - user_id="user1", environment_id="env1", source_kind="superset_link", - source_input="http://link", dataset_ref="dataset1" + user_id="user1", + environment_id="env1", + source_kind="superset_link", + source_input="http://link", + dataset_ref="dataset1", ) repo.create_session(session) - + profile = DatasetProfile( session_id=session.session_id, dataset_name="Test DS", business_summary="Summary", business_summary_source=BusinessSummarySource.INFERRED, - confidence_state=ConfidenceState.UNRESOLVED + confidence_state=ConfidenceState.UNRESOLVED, ) - + finding = ValidationFinding( session_id=session.session_id, area=FindingArea.SOURCE_INTAKE, severity=FindingSeverity.BLOCKING, code="ERR1", title="Error", - message="Failure" + message="Failure", ) - + repo.save_profile_and_findings(session.session_id, "user1", profile, [finding]) - + updated_session = repo.load_session_detail(session.session_id, "user1") assert updated_session.profile.dataset_name == "Test DS" assert len(updated_session.findings) == 1 assert updated_session.findings[0].code == "ERR1" - + # Verify removal of old findings new_finding = ValidationFinding( session_id=session.session_id, @@ -181,29 +223,34 @@ def test_save_profile_and_findings(db_session): severity=FindingSeverity.WARNING, code="WARN1", title="Warning", - message="Something" + message="Something", ) - + repo.save_profile_and_findings(session.session_id, "user1", profile, [new_finding]) - + db_session.expire_all() final_session = repo.load_session_detail(session.session_id, "user1") assert len(final_session.findings) == 1 assert final_session.findings[0].code == "WARN1" + # [/DEF:test_save_profile_and_findings:Function] + # [DEF:test_save_run_context:Function] # @RELATION: BINDS_TO -> SessionRepositoryTests def test_save_run_context(db_session): # @PURPOSE: Verify saving of run context. repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( - user_id="user1", environment_id="env1", source_kind="superset_link", - source_input="http://link", dataset_ref="dataset1" + user_id="user1", + environment_id="env1", + source_kind="superset_link", + source_input="http://link", + dataset_ref="dataset1", ) repo.create_session(session) - + rc = DatasetRunContext( session_id=session.session_id, dataset_ref="ds1", @@ -215,28 +262,50 @@ def test_save_run_context(db_session): approved_mapping_ids=[], semantic_decision_refs=[], open_warning_refs=[], - launch_status="success" + launch_status="success", ) repo.save_run_context(session.session_id, "user1", rc) - + assert session.last_run_context_id == rc.run_context_id + # [/DEF:test_save_run_context:Function] + # [DEF:test_list_sessions_for_user:Function] # @RELATION: BINDS_TO -> SessionRepositoryTests def test_list_sessions_for_user(db_session): # @PURPOSE: Verify listing of sessions by user. repo = DatasetReviewSessionRepository(db_session) - s1 = DatasetReviewSession(user_id="user1", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r1") - s2 = DatasetReviewSession(user_id="user1", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r2") - s3 = DatasetReviewSession(user_id="other", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r3") - + s1 = DatasetReviewSession( + user_id="user1", + environment_id="env1", + source_kind="k", + source_input="i", + dataset_ref="r1", + ) + s2 = DatasetReviewSession( + user_id="user1", + environment_id="env1", + source_kind="k", + source_input="i", + dataset_ref="r2", + ) + s3 = DatasetReviewSession( + user_id="other", + environment_id="env1", + source_kind="k", + source_input="i", + dataset_ref="r3", + ) + db_session.add_all([s1, s2, s3]) db_session.commit() - + sessions = repo.list_sessions_for_user("user1") assert len(sessions) == 2 assert all(s.user_id == "user1" for s in sessions) -# [/DEF:SessionRepositoryTests:Module]# [/DEF:test_list_sessions_for_user:Function] + +# [/DEF:test_list_sessions_for_user:Function] +# [/DEF:SessionRepositoryTests:Module] diff --git a/backend/src/services/dataset_review/repositories/session_repository.py b/backend/src/services/dataset_review/repositories/session_repository.py index dab7377d..b5703177 100644 --- a/backend/src/services/dataset_review/repositories/session_repository.py +++ b/backend/src/services/dataset_review/repositories/session_repository.py @@ -34,13 +34,14 @@ from src.core.logger import belief_scope, logger from src.services.dataset_review.event_logger import SessionEventLogger -# [DEF:SessionRepo:Class] +# [DEF:DatasetReviewSessionRepository:Class] # @COMPLEXITY: 4 # @PURPOSE: Enforce ownership-scoped persistence and retrieval for dataset review session aggregates. # @RELATION: [DEPENDS_ON] -> [DatasetReviewSession] # @RELATION: [DEPENDS_ON] -> [DatasetProfile] # @RELATION: [DEPENDS_ON] -> [ValidationFinding] # @RELATION: [DEPENDS_ON] -> [CompiledPreview] +# @RELATION: [DEPENDS_ON] -> [SessionEventLogger] # @PRE: constructor receives a live SQLAlchemy session and callers provide authenticated user scope for guarded reads and writes. # @POST: repository methods return ownership-scoped aggregates or persisted child records without changing domain meaning. # @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session. @@ -409,6 +410,6 @@ class DatasetReviewSessionRepository: # [/DEF:list_user_sess:Function] -# [/DEF:SessionRepo:Class] +# [/DEF:DatasetReviewSessionRepository:Class] # [/DEF:DatasetReviewSessionRepository:Module] diff --git a/backend/src/services/health_service.py b/backend/src/services/health_service.py index b549c0fa..9ae4b833 100644 --- a/backend/src/services/health_service.py +++ b/backend/src/services/health_service.py @@ -19,30 +19,44 @@ from ..core.superset_client import SupersetClient from ..core.task_manager.cleanup import TaskCleanupService from ..core.task_manager import TaskManager + # [DEF:HealthService:Class] # @COMPLEXITY: 4 # @PURPOSE: Aggregate latest dashboard validation state and manage persisted health report lifecycle. +# @PRE: Service is constructed with a live SQLAlchemy session and optional config manager. +# @POST: Exposes health summary aggregation and validation report deletion operations. +# @SIDE_EFFECT: Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators. +# @DATA_CONTRACT: Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool] +# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session] # @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord] # @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.DashboardHealthItem] # @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.HealthSummaryResponse] # @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient] # @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService] class HealthService: - _dashboard_summary_cache: Dict[str, Tuple[float, Dict[str, Dict[str, Optional[str]]]]] = {} + _dashboard_summary_cache: Dict[ + str, Tuple[float, Dict[str, Dict[str, Optional[str]]]] + ] = {} _dashboard_summary_cache_ttl_seconds = 60.0 """ @PURPOSE: Service for managing and querying dashboard health data. """ + # [DEF:HealthService_init:Function] # @COMPLEXITY: 3 # @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution. # @PRE: db is a valid SQLAlchemy session. # @POST: Service is ready to aggregate summaries and delete health reports. - def __init__(self, db: Session, config_manager = None): + # @SIDE_EFFECT: Initializes per-instance dashboard metadata cache. + # @DATA_CONTRACT: Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService] + # @RELATION: [BINDS_TO] ->[backend.src.services.health_service.HealthService] + # @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session] + def __init__(self, db: Session, config_manager=None): self.db = db self.config_manager = config_manager self._dashboard_meta_cache: Dict[Tuple[str, str], Dict[str, Optional[str]]] = {} + # [/DEF:HealthService_init:Function] # [DEF:_prime_dashboard_meta_cache:Function] @@ -51,6 +65,10 @@ class HealthService: # @PRE: records may contain mixed numeric and slug dashboard identifiers. # @POST: Numeric dashboard ids for known environments are cached when discoverable. # @SIDE_EFFECT: May call Superset dashboard list API once per referenced environment. + # @DATA_CONTRACT: Input[records: List[ValidationRecord]] -> Output[None] + # @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord] + # @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient] + # @RELATION: [CALLS] ->[config_manager.get_environments] # @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_dashboards_summary] def _prime_dashboard_meta_cache(self, records: List[ValidationRecord]) -> None: if not self.config_manager or not records: @@ -87,10 +105,13 @@ class HealthService: continue try: - cached_meta = self.__class__._dashboard_summary_cache.get(environment_id) + cached_meta = self.__class__._dashboard_summary_cache.get( + environment_id + ) cache_is_fresh = ( cached_meta is not None - and (time.monotonic() - cached_meta[0]) < self.__class__._dashboard_summary_cache_ttl_seconds + and (time.monotonic() - cached_meta[0]) + < self.__class__._dashboard_summary_cache_ttl_seconds ) if cache_is_fresh: dashboard_meta_map = cached_meta[1] @@ -109,9 +130,11 @@ class HealthService: dashboard_meta_map, ) for dashboard_id in dashboard_ids: - self._dashboard_meta_cache[(environment_id, dashboard_id)] = dashboard_meta_map.get( - dashboard_id, - {"slug": None, "title": None}, + self._dashboard_meta_cache[(environment_id, dashboard_id)] = ( + dashboard_meta_map.get( + dashboard_id, + {"slug": None, "title": None}, + ) ) except Exception as exc: logger.warning( @@ -124,6 +147,7 @@ class HealthService: "slug": None, "title": None, } + # [/DEF:_prime_dashboard_meta_cache:Function] # [DEF:_resolve_dashboard_meta:Function] @@ -131,7 +155,10 @@ class HealthService: # @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record. # @PRE: dashboard_id may be numeric or slug-like; environment_id may be empty. # @POST: Returns dict with `slug` and `title` keys, using cache when possible. - def _resolve_dashboard_meta(self, dashboard_id: str, environment_id: Optional[str]) -> Dict[str, Optional[str]]: + # @SIDE_EFFECT: Writes default cache entries for unresolved numeric dashboard ids. + def _resolve_dashboard_meta( + self, dashboard_id: str, environment_id: Optional[str] + ) -> Dict[str, Optional[str]]: normalized_dashboard_id = str(dashboard_id or "").strip() normalized_environment_id = str(environment_id or "").strip() if not normalized_dashboard_id: @@ -151,6 +178,7 @@ class HealthService: meta = {"slug": None, "title": None} self._dashboard_meta_cache[cache_key] = meta return meta + # [/DEF:_resolve_dashboard_meta:Function] # [DEF:get_health_summary:Function] @@ -162,7 +190,9 @@ class HealthService: # @DATA_CONTRACT: Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse] # @RELATION: [CALLS] ->[self._prime_dashboard_meta_cache] # @RELATION: [CALLS] ->[self._resolve_dashboard_meta] - async def get_health_summary(self, environment_id: str = None) -> HealthSummaryResponse: + async def get_health_summary( + self, environment_id: str = None + ) -> HealthSummaryResponse: """ @PURPOSE: Aggregates the latest validation status for all dashboards. @PRE: environment_id (optional) to filter by environment. @@ -170,23 +200,25 @@ class HealthService: """ # [REASON] We need the latest ValidationRecord for each unique dashboard_id. # We use a subquery to find the max timestamp per dashboard_id. - + subquery = self.db.query( ValidationRecord.dashboard_id, - func.max(ValidationRecord.timestamp).label("max_ts") + func.max(ValidationRecord.timestamp).label("max_ts"), ) if environment_id: - subquery = subquery.filter(ValidationRecord.environment_id == environment_id) + subquery = subquery.filter( + ValidationRecord.environment_id == environment_id + ) subquery = subquery.group_by(ValidationRecord.dashboard_id).subquery() query = self.db.query(ValidationRecord).join( subquery, - (ValidationRecord.dashboard_id == subquery.c.dashboard_id) & - (ValidationRecord.timestamp == subquery.c.max_ts) + (ValidationRecord.dashboard_id == subquery.c.dashboard_id) + & (ValidationRecord.timestamp == subquery.c.max_ts), ) - + records = query.all() - + self._prime_dashboard_meta_cache(records) items = [] @@ -208,27 +240,32 @@ class HealthService: status = "UNKNOWN" meta = self._resolve_dashboard_meta(rec.dashboard_id, rec.environment_id) - items.append(DashboardHealthItem( - record_id=rec.id, - dashboard_id=rec.dashboard_id, - dashboard_slug=meta.get("slug"), - dashboard_title=meta.get("title"), - environment_id=rec.environment_id or "unknown", - status=status, - last_check=rec.timestamp, - task_id=rec.task_id, - summary=rec.summary - )) + items.append( + DashboardHealthItem( + record_id=rec.id, + dashboard_id=rec.dashboard_id, + dashboard_slug=meta.get("slug"), + dashboard_title=meta.get("title"), + environment_id=rec.environment_id or "unknown", + status=status, + last_check=rec.timestamp, + task_id=rec.task_id, + summary=rec.summary, + ) + ) + + logger.info( + f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records." + ) - logger.info(f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records.") - return HealthSummaryResponse( items=items, pass_count=pass_count, warn_count=warn_count, fail_count=fail_count, - unknown_count=unknown_count + unknown_count=unknown_count, ) + # [/DEF:get_health_summary:Function] # [DEF:delete_validation_report:Function] @@ -238,9 +275,19 @@ class HealthService: # @POST: Returns True only when a matching record was deleted. # @SIDE_EFFECT: Deletes DB rows, optional screenshot file, and optional task/log persistence. # @DATA_CONTRACT: Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool] + # @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord] + # @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.TaskManager] + # @RELATION: [CALLS] ->[os.path.exists] + # @RELATION: [CALLS] ->[os.remove] # @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService.delete_task_with_logs] - def delete_validation_report(self, record_id: str, task_manager: Optional[TaskManager] = None) -> bool: - record = self.db.query(ValidationRecord).filter(ValidationRecord.id == record_id).first() + def delete_validation_report( + self, record_id: str, task_manager: Optional[TaskManager] = None + ) -> bool: + record = ( + self.db.query(ValidationRecord) + .filter(ValidationRecord.id == record_id) + .first() + ) if not record: return False @@ -250,7 +297,9 @@ class HealthService: if record.environment_id is None: peer_query = peer_query.filter(ValidationRecord.environment_id.is_(None)) else: - peer_query = peer_query.filter(ValidationRecord.environment_id == record.environment_id) + peer_query = peer_query.filter( + ValidationRecord.environment_id == record.environment_id + ) records_to_delete = peer_query.all() screenshot_paths = [ @@ -306,8 +355,10 @@ class HealthService: ) return True + # [/DEF:delete_validation_report:Function] - + + # [/DEF:HealthService:Class] # [/DEF:health_service:Module] diff --git a/backend/src/services/mapping_service.py b/backend/src/services/mapping_service.py index 8e9a10a4..ccdf9798 100644 --- a/backend/src/services/mapping_service.py +++ b/backend/src/services/mapping_service.py @@ -2,9 +2,14 @@ # # @SEMANTICS: service, mapping, fuzzy-matching, superset # @PURPOSE: Orchestrates database fetching and fuzzy matching suggestions. +# @COMPLEXITY: 3 # @LAYER: Service -# @RELATION: DEPENDS_ON -> backend.src.core.superset_client -# @RELATION: DEPENDS_ON -> backend.src.core.utils.matching +# @PRE: source/target environment identifiers are provided by caller. +# @POST: Exposes stateless mapping suggestion orchestration over configured environments. +# @SIDE_EFFECT: Performs remote metadata reads through Superset API clients. +# @DATA_CONTRACT: Input[source_env_id: str, target_env_id: str] -> Output[List[Dict]] +# @RELATION: DEPENDS_ON -> SupersetClient +# @RELATION: DEPENDS_ON -> suggest_mappings # # @INVARIANT: Suggestions are based on database names. @@ -15,57 +20,80 @@ from ..core.superset_client import SupersetClient from ..core.utils.matching import suggest_mappings # [/SECTION] + # [DEF:MappingService:Class] # @PURPOSE: Service for handling database mapping logic. +# @COMPLEXITY: 3 +# @PRE: config_manager exposes get_environments() with environment objects containing id. +# @POST: Provides client resolution and mapping suggestion methods. +# @SIDE_EFFECT: Instantiates Superset clients and performs upstream metadata reads. +# @DATA_CONTRACT: Input[config_manager] -> Output[List[Dict]] +# @RELATION: DEPENDS_ON -> SupersetClient +# @RELATION: DEPENDS_ON -> suggest_mappings class MappingService: - # [DEF:init:Function] # @PURPOSE: Initializes the mapping service with a config manager. + # @COMPLEXITY: 3 # @PRE: config_manager is provided. # @PARAM: config_manager (ConfigManager) - The configuration manager. # @POST: Service is initialized. + # @RELATION: DEPENDS_ON -> MappingService def __init__(self, config_manager): with belief_scope("MappingService.__init__"): self.config_manager = config_manager + # [/DEF:init:Function] # [DEF:_get_client:Function] # @PURPOSE: Helper to get an initialized SupersetClient for an environment. + # @COMPLEXITY: 3 # @PARAM: env_id (str) - The ID of the environment. # @PRE: environment must exist in config. # @POST: Returns an initialized SupersetClient. # @RETURN: SupersetClient - Initialized client. + # @RELATION: CALLS -> SupersetClient def _get_client(self, env_id: str) -> SupersetClient: with belief_scope("MappingService._get_client", f"env_id={env_id}"): envs = self.config_manager.get_environments() env = next((e for e in envs if e.id == env_id), None) if not env: raise ValueError(f"Environment {env_id} not found") - + return SupersetClient(env) + # [/DEF:_get_client:Function] # [DEF:get_suggestions:Function] # @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions. + # @COMPLEXITY: 3 # @PARAM: source_env_id (str) - Source environment ID. # @PARAM: target_env_id (str) - Target environment ID. # @PRE: Both environments must be accessible. # @POST: Returns fuzzy-matched database suggestions. # @RETURN: List[Dict] - Suggested mappings. - async def get_suggestions(self, source_env_id: str, target_env_id: str) -> List[Dict]: - with belief_scope("MappingService.get_suggestions", f"source={source_env_id}, target={target_env_id}"): + # @RELATION: CALLS -> _get_client + # @RELATION: CALLS -> suggest_mappings + async def get_suggestions( + self, source_env_id: str, target_env_id: str + ) -> List[Dict]: + with belief_scope( + "MappingService.get_suggestions", + f"source={source_env_id}, target={target_env_id}", + ): """ Get suggested mappings between two environments. """ source_client = self._get_client(source_env_id) target_client = self._get_client(target_env_id) - + source_dbs = source_client.get_databases_summary() target_dbs = target_client.get_databases_summary() - + return suggest_mappings(source_dbs, target_dbs) + # [/DEF:get_suggestions:Function] + # [/DEF:MappingService:Class] # [/DEF:mapping_service:Module] diff --git a/backend/src/services/notifications/providers.py b/backend/src/services/notifications/providers.py index 06c58635..70c0baf0 100644 --- a/backend/src/services/notifications/providers.py +++ b/backend/src/services/notifications/providers.py @@ -3,8 +3,17 @@ # @COMPLEXITY: 5 # @SEMANTICS: notifications, providers, smtp, slack, telegram, abstraction # @PURPOSE: Defines abstract base and concrete implementations for external notification delivery. -# @RELATION: IMPLEMENTS ->[NotificationService:Class] +# @RELATION: [DEPENDED_ON_BY] ->[NotificationService] +# @RELATION: [DEPENDS_ON] ->[NotificationProvider] +# @RELATION: [DEPENDS_ON] ->[SMTPProvider] +# @RELATION: [DEPENDS_ON] ->[TelegramProvider] +# @RELATION: [DEPENDS_ON] ->[SlackProvider] # @LAYER: Infra +# @PRE: Provider configuration dictionaries are supplied by trusted configuration sources. +# @POST: Each provider exposes async send contract returning boolean delivery outcome. +# @SIDE_EFFECT: Performs outbound network I/O to SMTP or HTTP endpoints. +# @DATA_CONTRACT: Input[target, subject, body, context?] -> Output[bool] +# @INVARIANT: Concrete providers preserve boolean send contract and swallow transport exceptions into False. # # @INVARIANT: Providers must be stateless and resilient to network failures. # @INVARIANT: Sensitive credentials must be handled via encrypted config. @@ -20,7 +29,11 @@ from ...core.logger import logger # [DEF:NotificationProvider:Class] +# @COMPLEXITY: 2 # @PURPOSE: Abstract base class for all notification providers. +# @RELATION: [DEPENDED_ON_BY] ->[SMTPProvider] +# @RELATION: [DEPENDED_ON_BY] ->[TelegramProvider] +# @RELATION: [DEPENDED_ON_BY] ->[SlackProvider] class NotificationProvider(ABC): @abstractmethod async def send( @@ -45,7 +58,9 @@ class NotificationProvider(ABC): # [DEF:SMTPProvider:Class] +# @COMPLEXITY: 3 # @PURPOSE: Delivers notifications via SMTP. +# @RELATION: [INHERITS] ->[NotificationProvider] class SMTPProvider(NotificationProvider): def __init__(self, config: Dict[str, Any]): self.host = config.get("host") @@ -88,7 +103,9 @@ class SMTPProvider(NotificationProvider): # [DEF:TelegramProvider:Class] +# @COMPLEXITY: 3 # @PURPOSE: Delivers notifications via Telegram Bot API. +# @RELATION: [INHERITS] ->[NotificationProvider] class TelegramProvider(NotificationProvider): def __init__(self, config: Dict[str, Any]): self.bot_token = config.get("bot_token") @@ -126,7 +143,9 @@ class TelegramProvider(NotificationProvider): # [DEF:SlackProvider:Class] +# @COMPLEXITY: 3 # @PURPOSE: Delivers notifications via Slack Webhooks or API. +# @RELATION: [INHERITS] ->[NotificationProvider] class SlackProvider(NotificationProvider): def __init__(self, config: Dict[str, Any]): self.webhook_url = config.get("webhook_url") diff --git a/backend/src/services/profile_service.py b/backend/src/services/profile_service.py index 15c99741..1f835190 100644 --- a/backend/src/services/profile_service.py +++ b/backend/src/services/profile_service.py @@ -86,9 +86,18 @@ class ProfileAuthorizationError(Exception): # [DEF:ProfileService:Class] -# @RELATION: DEPENDS_ON -> sqlalchemy.orm.Session +# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.repository.AuthRepository] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient] +# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_profile_lookup.SupersetAccountLookupAdapter] +# @RELATION: [DEPENDS_ON] ->[backend.src.models.profile.UserDashboardPreference] +# @RELATION: [CALLS] ->[backend.src.services.rbac_permission_catalog.discover_declared_permissions] # @COMPLEXITY: 5 # @PURPOSE: Implements profile preference read/update flow and Superset account lookup degradation strategy. +# @PRE: Caller provides authenticated User context for external service methods. +# @POST: Preference operations remain user-scoped and return normalized profile/lookup responses. +# @SIDE_EFFECT: Writes preference records and encrypted tokens; performs external account lookups when requested. +# @DATA_CONTRACT: Input[User,ProfilePreferenceUpdateRequest|SupersetAccountLookupRequest] -> Output[ProfilePreferenceResponse|SupersetAccountLookupResponse|bool] class ProfileService: # [DEF:init:Function] # @RELATION: BINDS_TO -> ProfileService diff --git a/backend/src/services/reports/__tests__/test_report_normalizer.py b/backend/src/services/reports/__tests__/test_report_normalizer.py index 2e38cea2..5562ed4e 100644 --- a/backend/src/services/reports/__tests__/test_report_normalizer.py +++ b/backend/src/services/reports/__tests__/test_report_normalizer.py @@ -14,6 +14,7 @@ from src.services.reports.normalizer import normalize_task_report # [DEF:test_unknown_type_maps_to_unknown_profile:Function] # @RELATION: BINDS_TO -> test_report_normalizer +# @PURPOSE: Ensure unknown plugin IDs map to unknown profile with populated summary and error context. def test_unknown_type_maps_to_unknown_profile(): task = Task( id="unknown-1", @@ -34,8 +35,10 @@ def test_unknown_type_maps_to_unknown_profile(): # [/DEF:test_unknown_type_maps_to_unknown_profile:Function] + # [DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function] # @RELATION: BINDS_TO -> test_report_normalizer +# @PURPOSE: Ensure missing result payload still yields visible report details with result placeholder. def test_partial_payload_keeps_report_visible_with_placeholders(): task = Task( id="partial-1", @@ -56,8 +59,10 @@ def test_partial_payload_keeps_report_visible_with_placeholders(): # [/DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function] + # [DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function] # @RELATION: BINDS_TO -> test_report_normalizer +# @PURPOSE: Ensure clean-release plugin ID maps to clean_release task profile and summary passthrough. def test_clean_release_plugin_maps_to_clean_release_task_type(): task = Task( id="clean-release-1", @@ -75,4 +80,5 @@ def test_clean_release_plugin_maps_to_clean_release_task_type(): assert report.summary == "Clean release compliance passed" -# [/DEF:test_report_normalizer:Module]# [/DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function] +# [/DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function] +# [/DEF:test_report_normalizer:Module] diff --git a/backend/src/services/reports/type_profiles.py b/backend/src/services/reports/type_profiles.py index 2f053b56..41e723de 100644 --- a/backend/src/services/reports/type_profiles.py +++ b/backend/src/services/reports/type_profiles.py @@ -3,7 +3,9 @@ # @SEMANTICS: reports, type_profiles, normalization, fallback # @PURPOSE: Deterministic mapping of plugin/task identifiers to canonical report task types and fallback profile metadata. # @LAYER: Domain -# @RELATION: DEPENDS_ON ->[backend.src.models.report.TaskType:Function] +# @RELATION: [DEPENDS_ON] ->[TaskType] +# @RELATION: [CONTAINS] ->[resolve_task_type] +# @RELATION: [CONTAINS] ->[get_type_profile] # @INVARIANT: Unknown input always resolves to TaskType.UNKNOWN with a single fallback profile. # [SECTION: IMPORTS] @@ -97,6 +99,8 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType: if not normalized: return TaskType.UNKNOWN return PLUGIN_TO_TASK_TYPE.get(normalized, TaskType.UNKNOWN) + + # [/DEF:resolve_task_type:Function] @@ -118,6 +122,8 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType: def get_type_profile(task_type: TaskType) -> Dict[str, Any]: with belief_scope("get_type_profile"): return TASK_TYPE_PROFILES.get(task_type, TASK_TYPE_PROFILES[TaskType.UNKNOWN]) + + # [/DEF:get_type_profile:Function] -# [/DEF:type_profiles:Module] \ No newline at end of file +# [/DEF:type_profiles:Module] diff --git a/backend/tests/core/migration/test_archive_parser.py b/backend/tests/core/migration/test_archive_parser.py index 613362d9..43a86e66 100644 --- a/backend/tests/core/migration/test_archive_parser.py +++ b/backend/tests/core/migration/test_archive_parser.py @@ -22,6 +22,7 @@ from src.core.migration.archive_parser import MigrationArchiveParser # [DEF:test_extract_objects_from_zip_collects_all_types:Function] # @RELATION: BINDS_TO -> TestArchiveParser +# @PURPOSE: Verify archive parser collects dashboard/chart/dataset YAML objects into typed buckets. def test_extract_objects_from_zip_collects_all_types(): parser = MigrationArchiveParser() with tempfile.TemporaryDirectory() as td: @@ -33,11 +34,19 @@ def test_extract_objects_from_zip_collects_all_types(): (src_dir / "datasets").mkdir(parents=True) with open(src_dir / "dashboards" / "dash.yaml", "w") as file_obj: - yaml.dump({"uuid": "dash-u1", "dashboard_title": "D1", "json_metadata": "{}"}, file_obj) + yaml.dump( + {"uuid": "dash-u1", "dashboard_title": "D1", "json_metadata": "{}"}, + file_obj, + ) with open(src_dir / "charts" / "chart.yaml", "w") as file_obj: - yaml.dump({"uuid": "chart-u1", "slice_name": "C1", "viz_type": "bar"}, file_obj) + yaml.dump( + {"uuid": "chart-u1", "slice_name": "C1", "viz_type": "bar"}, file_obj + ) with open(src_dir / "datasets" / "dataset.yaml", "w") as file_obj: - yaml.dump({"uuid": "ds-u1", "table_name": "orders", "database_uuid": "db-u1"}, file_obj) + yaml.dump( + {"uuid": "ds-u1", "table_name": "orders", "database_uuid": "db-u1"}, + file_obj, + ) with zipfile.ZipFile(zip_path, "w") as zip_obj: for root, _, files in os.walk(src_dir): @@ -61,5 +70,5 @@ def test_extract_objects_from_zip_collects_all_types(): raise AssertionError("dataset uuid mismatch") -# [/DEF:TestArchiveParser:Module] # [/DEF:test_extract_objects_from_zip_collects_all_types:Function] +# [/DEF:TestArchiveParser:Module] diff --git a/backend/tests/core/migration/test_dry_run_orchestrator.py b/backend/tests/core/migration/test_dry_run_orchestrator.py index 70f7167b..2d4069d4 100644 --- a/backend/tests/core/migration/test_dry_run_orchestrator.py +++ b/backend/tests/core/migration/test_dry_run_orchestrator.py @@ -3,7 +3,7 @@ # @COMPLEXITY: 3 # @PURPOSE: Unit tests for MigrationDryRunService diff and risk computation contracts. # @LAYER: Domain -# @RELATION: VERIFIES -> backend.src.core.migration.dry_run_orchestrator +# @RELATION: VERIFIES -> [MigrationDryRunOrchestratorModule] # import json import sys @@ -24,16 +24,21 @@ from src.models.mapping import Base # [DEF:_load_fixture:Function] -# @RELATION: BINDS_TO -> TestDryRunOrchestrator +# @RELATION: BINDS_TO -> [TestDryRunOrchestrator] +# @PURPOSE: Load canonical migration dry-run fixture payload used by deterministic orchestration assertions. def _load_fixture() -> dict: - fixture_path = Path(__file__).parents[2] / "fixtures" / "migration_dry_run_fixture.json" + fixture_path = ( + Path(__file__).parents[2] / "fixtures" / "migration_dry_run_fixture.json" + ) return json.loads(fixture_path.read_text()) # [/DEF:_load_fixture:Function] + # [DEF:_make_session:Function] -# @RELATION: BINDS_TO -> TestDryRunOrchestrator +# @RELATION: BINDS_TO -> [TestDryRunOrchestrator] +# @PURPOSE: Build isolated in-memory SQLAlchemy session for dry-run service tests. def _make_session(): engine = create_engine( "sqlite:///:memory:", @@ -47,8 +52,10 @@ def _make_session(): # [/DEF:_make_session:Function] + # [DEF:test_migration_dry_run_service_builds_diff_and_risk:Function] -# @RELATION: BINDS_TO -> TestDryRunOrchestrator +# @RELATION: BINDS_TO -> [TestDryRunOrchestrator] +# @PURPOSE: Verify dry-run orchestration returns stable diff summary and required risk codes. def test_migration_dry_run_service_builds_diff_and_risk(): # @TEST_CONTRACT: dry_run_result_contract -> { # required_fields: {diff: object, summary: object, risk: object}, @@ -68,7 +75,9 @@ def test_migration_dry_run_service_builds_diff_and_risk(): ) source_client = MagicMock() - source_client.get_dashboards_summary.return_value = fixture["source_dashboard_summary"] + source_client.get_dashboards_summary.return_value = fixture[ + "source_dashboard_summary" + ] source_client.export_dashboard.return_value = (b"PK\x03\x04", "source.zip") target_client = MagicMock() @@ -117,5 +126,5 @@ def test_migration_dry_run_service_builds_diff_and_risk(): raise AssertionError("breaking_reference risk is not detected") -# [/DEF:TestDryRunOrchestrator:Module] # [/DEF:test_migration_dry_run_service_builds_diff_and_risk:Function] +# [/DEF:TestDryRunOrchestrator:Module] diff --git a/backend/tests/core/test_migration_engine.py b/backend/tests/core/test_migration_engine.py index e63f0cf0..825d16ec 100644 --- a/backend/tests/core/test_migration_engine.py +++ b/backend/tests/core/test_migration_engine.py @@ -3,7 +3,7 @@ # @COMPLEXITY: 3 # @PURPOSE: Unit tests for MigrationEngine's cross-filter patching algorithms. # @LAYER: Domain -# @RELATION: VERIFIES -> backend.src.core.migration_engine +# @RELATION: VERIFIES -> [src.core.migration_engine:Module] # import pytest import tempfile @@ -28,7 +28,7 @@ from src.models.mapping import ResourceType # [DEF:MockMappingService:Class] -# @RELATION: BINDS_TO ->[TestMigrationEngine] +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] # @COMPLEXITY: 2 # @PURPOSE: Deterministic mapping service double for native filter ID remapping scenarios. # @INVARIANT: Returns mappings only for requested UUID keys present in seeded map. @@ -50,7 +50,8 @@ class MockMappingService: # [DEF:_write_dashboard_yaml:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Serialize dashboard metadata into YAML fixture with json_metadata payload for patch tests. def _write_dashboard_yaml(dir_path: Path, metadata: dict) -> Path: """Helper: writes a dashboard YAML file with json_metadata.""" file_path = dir_path / "dash.yaml" @@ -65,7 +66,8 @@ def _write_dashboard_yaml(dir_path: Path, metadata: dict) -> Path: # [DEF:test_patch_dashboard_metadata_replaces_chart_ids:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Verify native filter target chartId values are remapped via mapping service results. def test_patch_dashboard_metadata_replaces_chart_ids(): """Verifies that chartId values are replaced using the mapping service.""" mock_service = MockMappingService({"uuid-chart-A": 999}) @@ -91,7 +93,8 @@ def test_patch_dashboard_metadata_replaces_chart_ids(): # [DEF:test_patch_dashboard_metadata_replaces_dataset_ids:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Verify native filter target datasetId values are remapped via mapping service results. def test_patch_dashboard_metadata_replaces_dataset_ids(): """Verifies that datasetId values are replaced using the mapping service.""" mock_service = MockMappingService({"uuid-ds-B": 500}) @@ -118,7 +121,8 @@ def test_patch_dashboard_metadata_replaces_dataset_ids(): # [DEF:test_patch_dashboard_metadata_skips_when_no_metadata:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Ensure dashboard files without json_metadata are left unchanged by metadata patching. def test_patch_dashboard_metadata_skips_when_no_metadata(): """Verifies early return when json_metadata key is absent.""" mock_service = MockMappingService({}) @@ -140,7 +144,8 @@ def test_patch_dashboard_metadata_skips_when_no_metadata(): # [DEF:test_patch_dashboard_metadata_handles_missing_targets:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Verify patching updates mapped targets while preserving unmapped native filter IDs. def test_patch_dashboard_metadata_handles_missing_targets(): """When some source IDs have no target mapping, patches what it can and leaves the rest.""" mock_service = MockMappingService({"uuid-A": 100}) # Only uuid-A maps @@ -173,7 +178,8 @@ def test_patch_dashboard_metadata_handles_missing_targets(): # [DEF:test_extract_chart_uuids_from_archive:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Verify chart archive scan returns complete local chart id-to-uuid mapping. def test_extract_chart_uuids_from_archive(): """Verifies that chart YAML files are parsed for id->uuid mappings.""" engine = MigrationEngine() @@ -201,7 +207,8 @@ def test_extract_chart_uuids_from_archive(): # [DEF:test_transform_yaml_replaces_database_uuid:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Ensure dataset YAML database_uuid fields are replaced when source UUID mapping exists. def test_transform_yaml_replaces_database_uuid(): """Verifies that database_uuid in a dataset YAML is replaced.""" engine = MigrationEngine() @@ -223,7 +230,8 @@ def test_transform_yaml_replaces_database_uuid(): # [DEF:test_transform_yaml_ignores_unmapped_uuid:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Ensure transform_yaml leaves dataset files untouched when database_uuid is not mapped. def test_transform_yaml_ignores_unmapped_uuid(): """Verifies no changes when UUID is not in the mapping.""" engine = MigrationEngine() @@ -247,7 +255,8 @@ def test_transform_yaml_ignores_unmapped_uuid(): # [DEF:test_transform_zip_end_to_end:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Validate full ZIP transform pipeline remaps datasets and dashboard cross-filter chart IDs. def test_transform_zip_end_to_end(): """Verifies full orchestration: extraction, transformation, patching, and re-packaging.""" mock_service = MockMappingService({"char-uuid": 101, "ds-uuid": 202}) @@ -327,7 +336,8 @@ def test_transform_zip_end_to_end(): # [DEF:test_transform_zip_invalid_path:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Verify transform_zip returns False when source archive path does not exist. def test_transform_zip_invalid_path(): """@PRE: Verify behavior (False) on invalid ZIP path.""" engine = MigrationEngine() @@ -339,7 +349,8 @@ def test_transform_zip_invalid_path(): # [DEF:test_transform_yaml_nonexistent_file:Function] -# @RELATION: BINDS_TO -> TestMigrationEngine +# @RELATION: BINDS_TO -> [TestMigrationEngine:Module] +# @PURPOSE: Verify transform_yaml raises FileNotFoundError for missing YAML source files. def test_transform_yaml_nonexistent_file(): """@PRE: Verify behavior on non-existent YAML file.""" engine = MigrationEngine() @@ -349,5 +360,5 @@ def test_transform_yaml_nonexistent_file(): engine._transform_yaml(Path("non_existent.yaml"), {}) -# [/DEF:TestMigrationEngine:Module] # [/DEF:test_transform_yaml_nonexistent_file:Function] +# [/DEF:TestMigrationEngine:Module] diff --git a/backend/tests/scripts/test_clean_release_cli.py b/backend/tests/scripts/test_clean_release_cli.py index 113f551c..db4d4f39 100644 --- a/backend/tests/scripts/test_clean_release_cli.py +++ b/backend/tests/scripts/test_clean_release_cli.py @@ -13,13 +13,19 @@ from src.dependencies import get_clean_release_repository, get_config_manager from datetime import datetime, timezone from uuid import uuid4 -from src.models.clean_release import CleanPolicySnapshot, ComplianceReport, ReleaseCandidate, SourceRegistrySnapshot +from src.models.clean_release import ( + CleanPolicySnapshot, + ComplianceReport, + ReleaseCandidate, + SourceRegistrySnapshot, +) from src.services.clean_release.enums import CandidateStatus, ComplianceDecision from src.scripts.clean_release_cli import main as cli_main # [DEF:test_cli_candidate_register_scaffold:Function] # @RELATION: BINDS_TO -> test_clean_release_cli +# @PURPOSE: Verify candidate-register command exits successfully for valid required arguments. def test_cli_candidate_register_scaffold() -> None: """Candidate register CLI command smoke test.""" exit_code = cli_main( @@ -40,8 +46,10 @@ def test_cli_candidate_register_scaffold() -> None: # [/DEF:test_cli_candidate_register_scaffold:Function] + # [DEF:test_cli_manifest_build_scaffold:Function] # @RELATION: BINDS_TO -> test_clean_release_cli +# @PURPOSE: Verify candidate-register/artifact-import/manifest-build smoke path succeeds end-to-end. def test_cli_manifest_build_scaffold() -> None: """Manifest build CLI command smoke test.""" register_exit = cli_main( @@ -90,8 +98,10 @@ def test_cli_manifest_build_scaffold() -> None: # [/DEF:test_cli_manifest_build_scaffold:Function] + # [DEF:test_cli_compliance_run_scaffold:Function] # @RELATION: BINDS_TO -> test_clean_release_cli +# @PURPOSE: Verify compliance run/status/violations/report commands complete for prepared candidate. def test_cli_compliance_run_scaffold() -> None: """Compliance CLI command smoke test for run/status/report/violations.""" repository = get_clean_release_repository() @@ -119,6 +129,7 @@ def test_cli_compliance_run_scaffold() -> None: config = config_manager.get_config() if getattr(config, "settings", None) is None: + # @INVARIANT: SimpleNamespace substitutes for GlobalSettings — any field rename in GlobalSettings will silently not propagate here; re-verify on GlobalSettings schema changes. config.settings = SimpleNamespace() config.settings.clean_release = SimpleNamespace( active_policy_id=policy.id, @@ -180,7 +191,11 @@ def test_cli_compliance_run_scaffold() -> None: ) assert run_exit == 0 - run_id = next(run.id for run in repository.check_runs.values() if run.candidate_id == "cli-candidate-3") + run_id = next( + run.id + for run in repository.check_runs.values() + if run.candidate_id == "cli-candidate-3" + ) status_exit = cli_main(["compliance-status", "--run-id", run_id, "--json"]) assert status_exit == 0 @@ -194,8 +209,10 @@ def test_cli_compliance_run_scaffold() -> None: # [/DEF:test_cli_compliance_run_scaffold:Function] + # [DEF:test_cli_release_gate_commands_scaffold:Function] # @RELATION: BINDS_TO -> test_clean_release_cli +# @PURPOSE: Verify approve/reject/publish/revoke release-gate commands execute with valid fixtures. def test_cli_release_gate_commands_scaffold() -> None: """Release gate CLI smoke test for approve/reject/publish/revoke commands.""" repository = get_clean_release_repository() @@ -231,7 +248,11 @@ def test_cli_release_gate_commands_scaffold() -> None: run_id=f"run-{uuid4()}", candidate_id=approved_candidate_id, final_status=ComplianceDecision.PASSED.value, - summary_json={"operator_summary": "ok", "violations_count": 0, "blocking_violations_count": 0}, + summary_json={ + "operator_summary": "ok", + "violations_count": 0, + "blocking_violations_count": 0, + }, generated_at=datetime.now(timezone.utc), immutable=True, ) @@ -242,7 +263,11 @@ def test_cli_release_gate_commands_scaffold() -> None: run_id=f"run-{uuid4()}", candidate_id=rejected_candidate_id, final_status=ComplianceDecision.PASSED.value, - summary_json={"operator_summary": "ok", "violations_count": 0, "blocking_violations_count": 0}, + summary_json={ + "operator_summary": "ok", + "violations_count": 0, + "blocking_violations_count": 0, + }, generated_at=datetime.now(timezone.utc), immutable=True, ) @@ -317,5 +342,5 @@ def test_cli_release_gate_commands_scaffold() -> None: assert revoke_exit == 0 -# [/DEF:test_clean_release_cli:Module] # [/DEF:test_cli_release_gate_commands_scaffold:Function] +# [/DEF:test_clean_release_cli:Module] diff --git a/backend/tests/scripts/test_clean_release_tui_v2.py b/backend/tests/scripts/test_clean_release_tui_v2.py index 3faba721..73b75c26 100644 --- a/backend/tests/scripts/test_clean_release_tui_v2.py +++ b/backend/tests/scripts/test_clean_release_tui_v2.py @@ -17,6 +17,7 @@ from src.scripts.clean_release_tui import CleanReleaseTUI, main # [DEF:_build_mock_stdscr:Function] # @RELATION: BINDS_TO -> test_clean_release_tui_v2 +# @PURPOSE: Build deterministic curses screen mock with default terminal geometry and exit key. def _build_mock_stdscr() -> MagicMock: stdscr = MagicMock() stdscr.getmaxyx.return_value = (40, 120) @@ -26,9 +27,11 @@ def _build_mock_stdscr() -> MagicMock: # [/DEF:_build_mock_stdscr:Function] + @patch("src.scripts.clean_release_tui.curses") # [DEF:test_tui_f5_dispatches_run_action:Function] # @RELATION: BINDS_TO -> test_clean_release_tui_v2 +# @PURPOSE: Verify F5 key dispatch invokes run_checks exactly once before graceful exit. def test_tui_f5_dispatches_run_action(mock_curses_module: MagicMock) -> None: """F5 should dispatch run action from TUI loop.""" mock_curses_module.KEY_F10 = curses.KEY_F10 @@ -48,9 +51,11 @@ def test_tui_f5_dispatches_run_action(mock_curses_module: MagicMock) -> None: # [/DEF:test_tui_f5_dispatches_run_action:Function] + @patch("src.scripts.clean_release_tui.curses") # [DEF:test_tui_f5_run_smoke_reports_blocked_state:Function] # @RELATION: BINDS_TO -> test_clean_release_tui_v2 +# @PURPOSE: Verify blocked compliance state is surfaced after F5-triggered run action. def test_tui_f5_run_smoke_reports_blocked_state(mock_curses_module: MagicMock) -> None: """F5 smoke test should expose blocked outcome state after run action.""" mock_curses_module.KEY_F10 = curses.KEY_F10 @@ -77,8 +82,10 @@ def test_tui_f5_run_smoke_reports_blocked_state(mock_curses_module: MagicMock) - # [/DEF:test_tui_f5_run_smoke_reports_blocked_state:Function] + # [DEF:test_tui_non_tty_refuses_startup:Function] # @RELATION: BINDS_TO -> test_clean_release_tui_v2 +# @PURPOSE: Verify non-TTY execution returns exit code 2 with actionable stderr guidance. def test_tui_non_tty_refuses_startup(capsys) -> None: """Non-TTY startup must refuse TUI mode and redirect operator to CLI/API flow.""" with patch("sys.stdout.isatty", return_value=False): @@ -92,9 +99,11 @@ def test_tui_non_tty_refuses_startup(capsys) -> None: # [/DEF:test_tui_non_tty_refuses_startup:Function] + @patch("src.scripts.clean_release_tui.curses") # [DEF:test_tui_f8_blocked_without_facade_binding:Function] # @RELATION: BINDS_TO -> test_clean_release_tui_v2 +# @PURPOSE: Verify F8 path reports disabled action instead of mutating hidden facade state. def test_tui_f8_blocked_without_facade_binding(mock_curses_module: MagicMock) -> None: """F8 should not perform hidden state mutation when facade action is not bound.""" mock_curses_module.KEY_F10 = curses.KEY_F10 @@ -112,5 +121,5 @@ def test_tui_f8_blocked_without_facade_binding(mock_curses_module: MagicMock) -> assert "F8 disabled" in app.last_error -# [/DEF:test_clean_release_tui_v2:Module] # [/DEF:test_tui_f8_blocked_without_facade_binding:Function] +# [/DEF:test_clean_release_tui_v2:Module] diff --git a/backend/tests/services/clean_release/test_candidate_manifest_services.py b/backend/tests/services/clean_release/test_candidate_manifest_services.py index e544725a..dfdb9535 100644 --- a/backend/tests/services/clean_release/test_candidate_manifest_services.py +++ b/backend/tests/services/clean_release/test_candidate_manifest_services.py @@ -1,5 +1,5 @@ # [DEF:test_candidate_manifest_services:Module] -# @RELATION: BELONGS_TO -> SrcRoot +# @RELATION: BELONGS_TO -> [SrcRoot:Module] # @COMPLEXITY: 3 # @PURPOSE: Test lifecycle and manifest versioning for release candidates. # @LAYER: Tests @@ -9,12 +9,17 @@ from datetime import datetime, timezone from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from src.core.database import Base -from src.models.clean_release import ReleaseCandidate, DistributionManifest, CandidateArtifact +from src.models.clean_release import ( + ReleaseCandidate, + DistributionManifest, + CandidateArtifact, +) from src.services.clean_release.enums import CandidateStatus from src.services.clean_release.candidate_service import register_candidate from src.services.clean_release.manifest_service import build_manifest_snapshot from src.services.clean_release.repository import CleanReleaseRepository + @pytest.fixture def db_session(): engine = create_engine("sqlite:///:memory:") @@ -24,8 +29,10 @@ def db_session(): yield session session.close() + # [DEF:test_candidate_lifecycle_transitions:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify release candidate allows legal status transitions and rejects forbidden back-transitions. def test_candidate_lifecycle_transitions(db_session): """ @PURPOSE: Verify legal state transitions for ReleaseCandidate. @@ -36,7 +43,7 @@ def test_candidate_lifecycle_transitions(db_session): version="1.0.0", source_snapshot_ref="ref-1", created_by="operator", - status=CandidateStatus.DRAFT + status=CandidateStatus.DRAFT, ) db_session.add(candidate) db_session.commit() @@ -47,13 +54,17 @@ def test_candidate_lifecycle_transitions(db_session): # Invalid transition: PREPARED -> DRAFT (should raise IllegalTransitionError) from src.services.clean_release.exceptions import IllegalTransitionError + with pytest.raises(IllegalTransitionError, match="Forbidden transition"): candidate.transition_to(CandidateStatus.DRAFT) + # [/DEF:test_candidate_lifecycle_transitions:Function] + # [DEF:test_manifest_versioning_and_immutability:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify manifest versions increment monotonically and older snapshots remain queryable. def test_manifest_versioning_and_immutability(db_session): """ @PURPOSE: Verify manifest versioning and immutability invariants. @@ -70,7 +81,7 @@ def test_manifest_versioning_and_immutability(db_session): source_snapshot_ref="ref1", content_json={}, created_at=datetime.now(timezone.utc), - created_by="operator" + created_by="operator", ) db_session.add(m1) @@ -84,23 +95,34 @@ def test_manifest_versioning_and_immutability(db_session): source_snapshot_ref="ref1", content_json={}, created_at=datetime.now(timezone.utc), - created_by="operator" + created_by="operator", ) db_session.add(m2) db_session.commit() - latest = db_session.query(DistributionManifest).filter_by(candidate_id=candidate_id).order_by(DistributionManifest.manifest_version.desc()).first() + latest = ( + db_session.query(DistributionManifest) + .filter_by(candidate_id=candidate_id) + .order_by(DistributionManifest.manifest_version.desc()) + .first() + ) assert latest.manifest_version == 2 assert latest.id == "manifest-v2" - all_manifests = db_session.query(DistributionManifest).filter_by(candidate_id=candidate_id).all() + all_manifests = ( + db_session.query(DistributionManifest) + .filter_by(candidate_id=candidate_id) + .all() + ) assert len(all_manifests) == 2 # [/DEF:test_manifest_versioning_and_immutability:Function] + # [DEF:_valid_artifacts:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Provide canonical valid artifact payload used by candidate registration tests. def _valid_artifacts(): return [ { @@ -114,8 +136,10 @@ def _valid_artifacts(): # [/DEF:_valid_artifacts:Function] + # [DEF:test_register_candidate_rejects_duplicate_candidate_id:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify duplicate candidate_id registration is rejected by service invariants. def test_register_candidate_rejects_duplicate_candidate_id(): repository = CleanReleaseRepository() register_candidate( @@ -140,8 +164,10 @@ def test_register_candidate_rejects_duplicate_candidate_id(): # [/DEF:test_register_candidate_rejects_duplicate_candidate_id:Function] + # [DEF:test_register_candidate_rejects_malformed_artifact_input:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify candidate registration rejects artifact payloads missing required fields. def test_register_candidate_rejects_malformed_artifact_input(): repository = CleanReleaseRepository() bad_artifacts = [{"id": "art-1", "path": "bin/app", "size": 42}] # missing sha256 @@ -159,8 +185,10 @@ def test_register_candidate_rejects_malformed_artifact_input(): # [/DEF:test_register_candidate_rejects_malformed_artifact_input:Function] + # [DEF:test_register_candidate_rejects_empty_artifact_set:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify candidate registration rejects empty artifact collections. def test_register_candidate_rejects_empty_artifact_set(): repository = CleanReleaseRepository() @@ -177,8 +205,10 @@ def test_register_candidate_rejects_empty_artifact_set(): # [/DEF:test_register_candidate_rejects_empty_artifact_set:Function] + # [DEF:test_manifest_service_rebuild_creates_new_version:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify repeated manifest build creates a new incremented immutable version. def test_manifest_service_rebuild_creates_new_version(): repository = CleanReleaseRepository() register_candidate( @@ -190,8 +220,12 @@ def test_manifest_service_rebuild_creates_new_version(): artifacts=_valid_artifacts(), ) - first = build_manifest_snapshot(repository=repository, candidate_id="manifest-version-1", created_by="operator") - second = build_manifest_snapshot(repository=repository, candidate_id="manifest-version-1", created_by="operator") + first = build_manifest_snapshot( + repository=repository, candidate_id="manifest-version-1", created_by="operator" + ) + second = build_manifest_snapshot( + repository=repository, candidate_id="manifest-version-1", created_by="operator" + ) assert first.manifest_version == 1 assert second.manifest_version == 2 @@ -200,8 +234,10 @@ def test_manifest_service_rebuild_creates_new_version(): # [/DEF:test_manifest_service_rebuild_creates_new_version:Function] + # [DEF:test_manifest_service_existing_manifest_cannot_be_mutated:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify existing manifest snapshot remains immutable when rebuilding newer manifest version. def test_manifest_service_existing_manifest_cannot_be_mutated(): repository = CleanReleaseRepository() register_candidate( @@ -213,10 +249,18 @@ def test_manifest_service_existing_manifest_cannot_be_mutated(): artifacts=_valid_artifacts(), ) - created = build_manifest_snapshot(repository=repository, candidate_id="manifest-immutable-1", created_by="operator") + created = build_manifest_snapshot( + repository=repository, + candidate_id="manifest-immutable-1", + created_by="operator", + ) original_digest = created.manifest_digest - rebuilt = build_manifest_snapshot(repository=repository, candidate_id="manifest-immutable-1", created_by="operator") + rebuilt = build_manifest_snapshot( + repository=repository, + candidate_id="manifest-immutable-1", + created_by="operator", + ) old_manifest = repository.get_manifest(created.id) assert old_manifest is not None @@ -227,13 +271,20 @@ def test_manifest_service_existing_manifest_cannot_be_mutated(): # [/DEF:test_manifest_service_existing_manifest_cannot_be_mutated:Function] + # [DEF:test_manifest_service_rejects_missing_candidate:Function] -# @RELATION: BINDS_TO -> test_candidate_manifest_services +# @RELATION: BINDS_TO -> [test_candidate_manifest_services:Module] +# @PURPOSE: Verify manifest build fails with missing candidate identifier. def test_manifest_service_rejects_missing_candidate(): repository = CleanReleaseRepository() with pytest.raises(ValueError, match="not found"): - build_manifest_snapshot(repository=repository, candidate_id="missing-candidate", created_by="operator") + build_manifest_snapshot( + repository=repository, + candidate_id="missing-candidate", + created_by="operator", + ) + -# [/DEF:test_candidate_manifest_services:Module] # [/DEF:test_manifest_service_rejects_missing_candidate:Function] +# [/DEF:test_candidate_manifest_services:Module] diff --git a/backend/tests/services/clean_release/test_policy_resolution_service.py b/backend/tests/services/clean_release/test_policy_resolution_service.py index 353428ec..16c8e86c 100644 --- a/backend/tests/services/clean_release/test_policy_resolution_service.py +++ b/backend/tests/services/clean_release/test_policy_resolution_service.py @@ -3,9 +3,9 @@ # @SEMANTICS: clean-release, policy-resolution, trusted-snapshots, contracts # @PURPOSE: Verify trusted policy snapshot resolution contract and error guards. # @LAYER: Tests -# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.policy_resolution_service -# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository -# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.exceptions +# @RELATION: DEPENDS_ON -> [policy_resolution_service] +# @RELATION: DEPENDS_ON -> [repository] +# @RELATION: DEPENDS_ON -> [clean_release_exceptions] # @INVARIANT: Resolution uses only ConfigManager active IDs and rejects runtime override attempts. from __future__ import annotations @@ -16,25 +16,33 @@ import pytest from src.models.clean_release import CleanPolicySnapshot, SourceRegistrySnapshot from src.services.clean_release.exceptions import PolicyResolutionError -from src.services.clean_release.policy_resolution_service import resolve_trusted_policy_snapshots +from src.services.clean_release.policy_resolution_service import ( + resolve_trusted_policy_snapshots, +) from src.services.clean_release.repository import CleanReleaseRepository # [DEF:_config_manager:Function] -# @RELATION: BINDS_TO -> TestPolicyResolutionService +# @RELATION: BINDS_TO -> [TestPolicyResolutionService] +# @COMPLEXITY: 1 # @PURPOSE: Build deterministic ConfigManager-like stub for tests. +# @INVARIANT: Only settings.clean_release.active_policy_id and active_registry_id are populated; any other settings field access raises AttributeError. # @PRE: policy_id and registry_id may be None or non-empty strings. # @POST: Returns object exposing get_config().settings.clean_release active IDs. def _config_manager(policy_id, registry_id): - clean_release = SimpleNamespace(active_policy_id=policy_id, active_registry_id=registry_id) + clean_release = SimpleNamespace( + active_policy_id=policy_id, active_registry_id=registry_id + ) settings = SimpleNamespace(clean_release=clean_release) config = SimpleNamespace(settings=settings) return SimpleNamespace(get_config=lambda: config) + + # [/DEF:_config_manager:Function] # [DEF:test_resolve_trusted_policy_snapshots_missing_profile:Function] -# @RELATION: BINDS_TO -> TestPolicyResolutionService +# @RELATION: BINDS_TO -> [TestPolicyResolutionService] # @PURPOSE: Ensure resolution fails when trusted profile is not configured. # @PRE: active_policy_id is None. # @POST: Raises PolicyResolutionError with missing trusted profile reason. @@ -47,11 +55,13 @@ def test_resolve_trusted_policy_snapshots_missing_profile(): config_manager=config_manager, repository=repository, ) + + # [/DEF:test_resolve_trusted_policy_snapshots_missing_profile:Function] # [DEF:test_resolve_trusted_policy_snapshots_missing_registry:Function] -# @RELATION: BINDS_TO -> TestPolicyResolutionService +# @RELATION: BINDS_TO -> [TestPolicyResolutionService] # @PURPOSE: Ensure resolution fails when trusted registry is not configured. # @PRE: active_registry_id is None and active_policy_id is set. # @POST: Raises PolicyResolutionError with missing trusted registry reason. @@ -64,11 +74,13 @@ def test_resolve_trusted_policy_snapshots_missing_registry(): config_manager=config_manager, repository=repository, ) + + # [/DEF:test_resolve_trusted_policy_snapshots_missing_registry:Function] # [DEF:test_resolve_trusted_policy_snapshots_rejects_override_attempt:Function] -# @RELATION: BINDS_TO -> TestPolicyResolutionService +# @RELATION: BINDS_TO -> [TestPolicyResolutionService] # @PURPOSE: Ensure runtime override attempt is rejected even if snapshots exist. # @PRE: valid trusted snapshots exist in repository and override is provided. # @POST: Raises PolicyResolutionError with override forbidden reason. @@ -104,6 +116,8 @@ def test_resolve_trusted_policy_snapshots_rejects_override_attempt(): repository=repository, policy_id_override="policy-override", ) + + # [/DEF:test_resolve_trusted_policy_snapshots_rejects_override_attempt:Function] # [/DEF:TestPolicyResolutionService:Module] diff --git a/backend/tests/test_resource_hubs.py b/backend/tests/test_resource_hubs.py index 0d8610be..e46267dd 100644 --- a/backend/tests/test_resource_hubs.py +++ b/backend/tests/test_resource_hubs.py @@ -1,42 +1,71 @@ +# [DEF:TestResourceHubs:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, resource-hubs, dashboards, datasets, pagination, api +# @PURPOSE: Contract tests for resource hub dashboards/datasets listing and pagination boundary validation. +# @LAYER: Domain (Tests) import pytest from fastapi.testclient import TestClient from unittest.mock import MagicMock, AsyncMock from src.app import app -from src.dependencies import get_config_manager, get_task_manager, get_resource_service, has_permission +from src.dependencies import ( + get_config_manager, + get_task_manager, + get_resource_service, + has_permission, +) client = TestClient(app) # [DEF:test_dashboards_api:Test] -# @RELATION: BINDS_TO -> SrcRoot +# @RELATION: BINDS_TO -> TestResourceHubs # @PURPOSE: Verify GET /api/dashboards contract compliance # @TEST: Valid env_id returns 200 and dashboard list # @TEST: Invalid env_id returns 404 # @TEST: Search filter works + @pytest.fixture def mock_deps(): + # @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass config_manager = MagicMock() + # @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass task_manager = MagicMock() + # @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass resource_service = MagicMock() - + # Mock environment env = MagicMock() env.id = "env1" config_manager.get_environments.return_value = [env] - + # Mock tasks task_manager.get_all_tasks.return_value = [] - + # Mock dashboards - resource_service.get_dashboards_with_status = AsyncMock(return_value=[ - {"id": 1, "title": "Sales", "slug": "sales", "git_status": {"branch": "main", "sync_status": "OK"}, "last_task": None}, - {"id": 2, "title": "Marketing", "slug": "mkt", "git_status": None, "last_task": {"task_id": "t1", "status": "SUCCESS"}} - ]) - + resource_service.get_dashboards_with_status = AsyncMock( + return_value=[ + { + "id": 1, + "title": "Sales", + "slug": "sales", + "git_status": {"branch": "main", "sync_status": "OK"}, + "last_task": None, + }, + { + "id": 2, + "title": "Marketing", + "slug": "mkt", + "git_status": None, + "last_task": {"task_id": "t1", "status": "SUCCESS"}, + }, + ] + ) + app.dependency_overrides[get_config_manager] = lambda: config_manager app.dependency_overrides[get_task_manager] = lambda: task_manager app.dependency_overrides[get_resource_service] = lambda: resource_service - + # Bypass permission check mock_user = MagicMock() mock_user.username = "testadmin" @@ -44,24 +73,25 @@ def mock_deps(): admin_role = MagicMock() admin_role.name = "Admin" mock_user.roles.append(admin_role) - + # Override both get_current_user and has_permission from src.dependencies import get_current_user + app.dependency_overrides[get_current_user] = lambda: mock_user - + # We need to override the specific instance returned by has_permission - app.dependency_overrides[has_permission("plugin:migration", "READ")] = lambda: mock_user - - yield { - "config": config_manager, - "task": task_manager, - "resource": resource_service - } - + app.dependency_overrides[has_permission("plugin:migration", "READ")] = ( + lambda: mock_user + ) + + yield {"config": config_manager, "task": task_manager, "resource": resource_service} + app.dependency_overrides.clear() + # [DEF:test_get_dashboards_success:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_dashboards_api +# @PURPOSE: Verify dashboards endpoint returns 200 with expected dashboard payload fields. def test_get_dashboards_success(mock_deps): response = client.get("/api/dashboards?env_id=env1") assert response.status_code == 200 @@ -71,18 +101,24 @@ def test_get_dashboards_success(mock_deps): assert data["dashboards"][0]["title"] == "Sales" assert data["dashboards"][0]["git_status"]["sync_status"] == "OK" + # [/DEF:test_get_dashboards_success:Function] + # [DEF:test_get_dashboards_not_found:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_dashboards_api +# @PURPOSE: Verify dashboards endpoint returns 404 for unknown environment identifier. def test_get_dashboards_not_found(mock_deps): response = client.get("/api/dashboards?env_id=invalid") assert response.status_code == 404 + # [/DEF:test_get_dashboards_not_found:Function] + # [DEF:test_get_dashboards_search:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_dashboards_api +# @PURPOSE: Verify dashboards endpoint search filter returns matching subset. def test_get_dashboards_search(mock_deps): response = client.get("/api/dashboards?env_id=env1&search=Sales") assert response.status_code == 200 @@ -90,25 +126,37 @@ def test_get_dashboards_search(mock_deps): assert len(data["dashboards"]) == 1 assert data["dashboards"][0]["title"] == "Sales" + +# [/DEF:test_get_dashboards_search:Function] # [/DEF:test_dashboards_api:Test] + # [DEF:test_datasets_api:Test] -# @RELATION: BINDS_TO -> SrcRoot +# @RELATION: BINDS_TO -> TestResourceHubs # @PURPOSE: Verify GET /api/datasets contract compliance # @TEST: Valid env_id returns 200 and dataset list # @TEST: Invalid env_id returns 404 # @TEST: Search filter works # @TEST: Negative - Service failure returns 503 -# [/DEF:test_get_dashboards_search:Function] # [DEF:test_get_datasets_success:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_datasets_api +# @PURPOSE: Verify datasets endpoint returns 200 with mapped fields payload. def test_get_datasets_success(mock_deps): - mock_deps["resource"].get_datasets_with_status = AsyncMock(return_value=[ - {"id": 1, "table_name": "orders", "schema": "public", "database": "db1", "mapped_fields": {"total": 10, "mapped": 5}, "last_task": None} - ]) - + mock_deps["resource"].get_datasets_with_status = AsyncMock( + return_value=[ + { + "id": 1, + "table_name": "orders", + "schema": "public", + "database": "db1", + "mapped_fields": {"total": 10, "mapped": 5}, + "last_task": None, + } + ] + ) + response = client.get("/api/datasets?env_id=env1") assert response.status_code == 200 data = response.json() @@ -117,86 +165,126 @@ def test_get_datasets_success(mock_deps): assert data["datasets"][0]["table_name"] == "orders" assert data["datasets"][0]["mapped_fields"]["mapped"] == 5 + # [/DEF:test_get_datasets_success:Function] + # [DEF:test_get_datasets_not_found:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_datasets_api +# @PURPOSE: Verify datasets endpoint returns 404 for unknown environment identifier. def test_get_datasets_not_found(mock_deps): response = client.get("/api/datasets?env_id=invalid") assert response.status_code == 404 + # [/DEF:test_get_datasets_not_found:Function] + # [DEF:test_get_datasets_search:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_datasets_api +# @PURPOSE: Verify datasets endpoint search filter returns matching dataset subset. def test_get_datasets_search(mock_deps): - mock_deps["resource"].get_datasets_with_status = AsyncMock(return_value=[ - {"id": 1, "table_name": "orders", "schema": "public", "database": "db1", "mapped_fields": {"total": 10, "mapped": 5}, "last_task": None}, - {"id": 2, "table_name": "users", "schema": "public", "database": "db1", "mapped_fields": {"total": 5, "mapped": 5}, "last_task": None} - ]) - + mock_deps["resource"].get_datasets_with_status = AsyncMock( + return_value=[ + { + "id": 1, + "table_name": "orders", + "schema": "public", + "database": "db1", + "mapped_fields": {"total": 10, "mapped": 5}, + "last_task": None, + }, + { + "id": 2, + "table_name": "users", + "schema": "public", + "database": "db1", + "mapped_fields": {"total": 5, "mapped": 5}, + "last_task": None, + }, + ] + ) + response = client.get("/api/datasets?env_id=env1&search=orders") assert response.status_code == 200 data = response.json() assert len(data["datasets"]) == 1 assert data["datasets"][0]["table_name"] == "orders" + # [/DEF:test_get_datasets_search:Function] + # [DEF:test_get_datasets_service_failure:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_datasets_api +# @PURPOSE: Verify datasets endpoint surfaces backend fetch failure as HTTP 503. def test_get_datasets_service_failure(mock_deps): - mock_deps["resource"].get_datasets_with_status = AsyncMock(side_effect=Exception("Superset down")) - + mock_deps["resource"].get_datasets_with_status = AsyncMock( + side_effect=Exception("Superset down") + ) + response = client.get("/api/datasets?env_id=env1") assert response.status_code == 503 assert "Failed to fetch datasets" in response.json()["detail"] + +# [/DEF:test_get_datasets_service_failure:Function] # [/DEF:test_datasets_api:Test] # [DEF:test_pagination_boundaries:Test] -# @RELATION: BINDS_TO -> SrcRoot +# @RELATION: BINDS_TO -> TestResourceHubs # @PURPOSE: Verify pagination validation for GET endpoints # @TEST: page<1 and page_size>100 return 400 -# [/DEF:test_get_datasets_service_failure:Function] # [DEF:test_get_dashboards_pagination_zero_page:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_pagination_boundaries +# @PURPOSE: Verify dashboards endpoint rejects page=0 with HTTP 400 validation error. def test_get_dashboards_pagination_zero_page(mock_deps): """@TEST_EDGE: pagination_zero_page -> {page:0, status:400}""" response = client.get("/api/dashboards?env_id=env1&page=0") assert response.status_code == 400 assert "Page must be >= 1" in response.json()["detail"] + # [/DEF:test_get_dashboards_pagination_zero_page:Function] + # [DEF:test_get_dashboards_pagination_oversize:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_pagination_boundaries +# @PURPOSE: Verify dashboards endpoint rejects oversized page_size with HTTP 400. def test_get_dashboards_pagination_oversize(mock_deps): """@TEST_EDGE: pagination_oversize -> {page_size:101, status:400}""" response = client.get("/api/dashboards?env_id=env1&page_size=101") assert response.status_code == 400 assert "Page size must be between 1 and 100" in response.json()["detail"] + # [/DEF:test_get_dashboards_pagination_oversize:Function] + # [DEF:test_get_datasets_pagination_zero_page:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_pagination_boundaries +# @PURPOSE: Verify datasets endpoint rejects page=0 with HTTP 400. def test_get_datasets_pagination_zero_page(mock_deps): """@TEST_EDGE: pagination_zero_page on datasets""" response = client.get("/api/datasets?env_id=env1&page=0") assert response.status_code == 400 + # [/DEF:test_get_datasets_pagination_zero_page:Function] + # [DEF:test_get_datasets_pagination_oversize:Function] -# @RELATION: BINDS_TO -> UnknownModule +# @RELATION: BINDS_TO -> test_pagination_boundaries +# @PURPOSE: Verify datasets endpoint rejects oversized page_size with HTTP 400. def test_get_datasets_pagination_oversize(mock_deps): """@TEST_EDGE: pagination_oversize on datasets""" response = client.get("/api/datasets?env_id=env1&page_size=101") assert response.status_code == 400 -# [/DEF:test_pagination_boundaries:Test] + # [/DEF:test_get_datasets_pagination_oversize:Function] +# [/DEF:test_pagination_boundaries:Test] +# [/DEF:TestResourceHubs:Module] diff --git a/frontend/src/components/Navbar.svelte b/frontend/src/components/Navbar.svelte index a9fb4092..64cc62ca 100644 --- a/frontend/src/components/Navbar.svelte +++ b/frontend/src/components/Navbar.svelte @@ -6,7 +6,7 @@ @SEMANTICS: navbar, navigation, header, layout @PURPOSE: Main navigation bar for the application. @LAYER: UI -@RELATION: USES -> AppState +@RELATION: USES -> auth -->