busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

semantics

Browse Source
This commit is contained in:
busya
2026-03-27 21:27:31 +03:00
parent 7c85552132
commit 2ed66bfebc
182 changed files with 21186 additions and 10254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
backend/src/api/routes/__init__.py
View File

@@ -4,9 +4,41 @@
# @PURPOSE: Provide lazy route module loading to avoid heavyweight imports during tests.
# @LAYER: API
# @RELATION: [CALLS] ->[ApiRoutesGetAttr]
# @RELATION: [BINDS_TO] ->[Route_Group_Contracts]
# @INVARIANT: Only names listed in __all__ are importable via __getattr__.
__all__ = ['plugins', 'tasks', 'settings', 'connections', 'environments', 'mappings', 'migration', 'git', 'storage', 'admin', 'reports', 'assistant', 'clean_release', 'profile', 'dataset_review']
# [DEF:Route_Group_Contracts:Block]
# @COMPLEXITY: 3
# @PURPOSE: Declare the canonical route-module registry used by lazy imports and app router inclusion.
# @RELATION: DEPENDS_ON -> [PluginsRouter]
# @RELATION: DEPENDS_ON -> [TasksRouter]
# @RELATION: DEPENDS_ON -> [SettingsRouter]
# @RELATION: DEPENDS_ON -> [ConnectionsRouter]
# @RELATION: DEPENDS_ON -> [ReportsRouter]
# @RELATION: DEPENDS_ON -> [LlmRoutes]
__all__ = [
"plugins",
"tasks",
"settings",
"connections",
"environments",
"mappings",
"migration",
"git",
"storage",
"admin",
"reports",
"assistant",
"clean_release",
"clean_release_v2",
"profile",
"dataset_review",
"llm",
"dashboards",
"datasets",
"health",
]
# [/DEF:Route_Group_Contracts:Block]
# [DEF:ApiRoutesGetAttr:Function]
@@ -18,7 +50,10 @@ __all__ = ['plugins', 'tasks', 'settings', 'connections', 'environments', 'mappi
def __getattr__(name):
if name in __all__:
import importlib
return importlib.import_module(f".{name}", __name__)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
# [/DEF:ApiRoutesGetAttr:Function]
# [/DEF:ApiRoutesModule:Module]

2
backend/src/api/routes/__tests__/test_assistant_api.py
View File

@@ -559,6 +559,4 @@ def test_dataset_review_scoped_command_routes_field_semantics_update():
# [/DEF:test_dataset_review_scoped_command_routes_field_semantics_update:Function]
# [/DEF:test_capabilities_question_returns_successful_help:Function]
# [/DEF:AssistantApiTests:Module]

12
backend/src/api/routes/clean_release_v2.py
View File

@@ -1,6 +1,13 @@
# [DEF:CleanReleaseV2Api:Module]
# @COMPLEXITY: 4
# @PURPOSE: Redesigned clean release API for headless candidate lifecycle.
# @LAYER: UI (API)
# @RELATION: DEPENDS_ON -> [CleanReleaseRepository]
# @RELATION: CALLS -> [approve_candidate]
# @RELATION: CALLS -> [publish_candidate]
# @PRE: Clean release repository dependency is available for candidate lifecycle endpoints.
# @POST: Candidate registration, approval, publication, and revocation routes are registered without behavior changes.
# @SIDE_EFFECT: Persists candidate lifecycle state through clean release services and repository adapters.
from fastapi import APIRouter, Depends, HTTPException, status
from typing import List, Dict, Any
@@ -29,7 +36,6 @@ router = APIRouter(prefix="/api/v2/clean-release", tags=["Clean Release V2"])
# [DEF:ApprovalRequest:Class]
# @COMPLEXITY: 1
# @PURPOSE: Schema for approval request payload.
# @RELATION: USES -> [CandidateDTO]
class ApprovalRequest(dict):
pass
@@ -40,7 +46,6 @@ class ApprovalRequest(dict):
# [DEF:PublishRequest:Class]
# @COMPLEXITY: 1
# @PURPOSE: Schema for publication request payload.
# @RELATION: USES -> [CandidateDTO]
class PublishRequest(dict):
pass
@@ -51,7 +56,6 @@ class PublishRequest(dict):
# [DEF:RevokeRequest:Class]
# @COMPLEXITY: 1
# @PURPOSE: Schema for revocation request payload.
# @RELATION: USES -> [CandidateDTO]
class RevokeRequest(dict):
pass
@@ -66,7 +70,7 @@ class RevokeRequest(dict):
# @POST: Candidate is saved in repository.
# @RETURN: CandidateDTO
# @RELATION: CALLS -> [CleanReleaseRepository.save_candidate]
# @RELATION: USES -> [CandidateDTO]
# @RELATION: DEPENDS_ON -> [CandidateDTO]
@router.post(
"/candidates", response_model=CandidateDTO, status_code=status.HTTP_201_CREATED
)

32
backend/src/api/routes/llm.py
View File

@@ -1,8 +1,12 @@
# [DEF:backend/src/api/routes/llm.py:Module]
# @COMPLEXITY: 2
# [DEF:LlmRoutes:Module]
# @COMPLEXITY: 3
# @SEMANTICS: api, routes, llm
# @PURPOSE: API routes for LLM provider configuration and management.
# @LAYER: UI (API)
# @RELATION: DEPENDS_ON -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [LLMProviderConfig]
# @RELATION: DEPENDS_ON -> [get_current_user]
# @RELATION: DEPENDS_ON -> [get_db]
from fastapi import APIRouter, Depends, HTTPException, status
from typing import List, Optional
@@ -24,6 +28,7 @@ router = APIRouter(tags=["LLM"])
# @PURPOSE: Validate decrypted runtime API key presence/shape.
# @PRE: value can be None.
# @POST: Returns True only for non-placeholder key.
# @RELATION: BINDS_TO -> [LlmRoutes]
def _is_valid_runtime_api_key(value: Optional[str]) -> bool:
key = (value or "").strip()
if not key:
@@ -40,6 +45,8 @@ def _is_valid_runtime_api_key(value: Optional[str]) -> bool:
# @PURPOSE: Retrieve all LLM provider configurations.
# @PRE: User is authenticated.
# @POST: Returns list of LLMProviderConfig.
# @RELATION: CALLS -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [LLMProviderConfig]
@router.get("/providers", response_model=List[LLMProviderConfig])
async def get_providers(
current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)
@@ -73,6 +80,8 @@ async def get_providers(
# @PURPOSE: Returns whether LLM runtime is configured for dashboard validation.
# @PRE: User is authenticated.
# @POST: configured=true only when an active provider with valid decrypted key exists.
# @RELATION: CALLS -> [LLMProviderService]
# @RELATION: CALLS -> [_is_valid_runtime_api_key]
@router.get("/status")
async def get_llm_status(
current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)
@@ -112,7 +121,9 @@ async def get_llm_status(
"configured": False,
"reason": "invalid_api_key",
"provider_count": len(providers),
"active_provider_count": len([provider for provider in providers if provider.is_active]),
"active_provider_count": len(
[provider for provider in providers if provider.is_active]
),
"provider_id": active_provider.id,
"provider_name": active_provider.name,
"provider_type": active_provider.provider_type,
@@ -123,7 +134,9 @@ async def get_llm_status(
"configured": True,
"reason": "ok",
"provider_count": len(providers),
"active_provider_count": len([provider for provider in providers if provider.is_active]),
"active_provider_count": len(
[provider for provider in providers if provider.is_active]
),
"provider_id": active_provider.id,
"provider_name": active_provider.name,
"provider_type": active_provider.provider_type,
@@ -138,6 +151,8 @@ async def get_llm_status(
# @PURPOSE: Create a new LLM provider configuration.
# @PRE: User is authenticated and has admin permissions.
# @POST: Returns the created LLMProviderConfig.
# @RELATION: CALLS -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [LLMProviderConfig]
@router.post(
"/providers", response_model=LLMProviderConfig, status_code=status.HTTP_201_CREATED
)
@@ -169,6 +184,8 @@ async def create_provider(
# @PURPOSE: Update an existing LLM provider configuration.
# @PRE: User is authenticated and has admin permissions.
# @POST: Returns the updated LLMProviderConfig.
# @RELATION: CALLS -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [LLMProviderConfig]
@router.put("/providers/{provider_id}", response_model=LLMProviderConfig)
async def update_provider(
provider_id: str,
@@ -202,6 +219,7 @@ async def update_provider(
# @PURPOSE: Delete an LLM provider configuration.
# @PRE: User is authenticated and has admin permissions.
# @POST: Returns success status.
# @RELATION: CALLS -> [LLMProviderService]
@router.delete("/providers/{provider_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_provider(
provider_id: str,
@@ -224,6 +242,8 @@ async def delete_provider(
# @PURPOSE: Test connection to an LLM provider.
# @PRE: User is authenticated.
# @POST: Returns success status and message.
# @RELATION: CALLS -> [LLMProviderService]
# @RELATION: DEPENDS_ON -> [LLMClient]
@router.post("/providers/{provider_id}/test")
async def test_connection(
provider_id: str,
@@ -276,6 +296,8 @@ async def test_connection(
# @PURPOSE: Test connection with a provided configuration (not yet saved).
# @PRE: User is authenticated.
# @POST: Returns success status and message.
# @RELATION: DEPENDS_ON -> [LLMClient]
# @RELATION: DEPENDS_ON -> [LLMProviderConfig]
@router.post("/providers/test")
async def test_provider_config(
config: LLMProviderConfig, current_user: User = Depends(get_current_active_user)
@@ -311,4 +333,4 @@ async def test_provider_config(
# [/DEF:test_provider_config:Function]
# [/DEF:backend/src/api/routes/llm.py:Module]
# [/DEF:LlmRoutes:Module]

15
backend/src/api/routes/plugins.py
View File

@@ -3,7 +3,9 @@
# @SEMANTICS: api, router, plugins, list
# @PURPOSE: Defines the FastAPI router for plugin-related endpoints, allowing clients to list available plugins.
# @LAYER: UI (API)
# @RELATION: Depends on the PluginLoader and PluginConfig. It is included by the main app.
# @RELATION: DEPENDS_ON -> [PluginConfig]
# @RELATION: DEPENDS_ON -> [get_plugin_loader]
# @RELATION: BINDS_TO -> [API_Routes]
from typing import List
from fastapi import APIRouter, Depends
@@ -13,20 +15,25 @@ from ...core.logger import belief_scope
router = APIRouter()
# [DEF:list_plugins:Function]
# @PURPOSE: Retrieve a list of all available plugins.
# @PRE: plugin_loader is injected via Depends.
# @POST: Returns a list of PluginConfig objects.
# @RETURN: List[PluginConfig] - List of registered plugins.
# @RELATION: CALLS -> [get_plugin_loader]
# @RELATION: DEPENDS_ON -> [PluginConfig]
@router.get("", response_model=List[PluginConfig])
async def list_plugins(
plugin_loader = Depends(get_plugin_loader),
_ = Depends(has_permission("plugins", "READ"))
plugin_loader=Depends(get_plugin_loader),
_=Depends(has_permission("plugins", "READ")),
):
with belief_scope("list_plugins"):
"""
Retrieve a list of all available plugins.
"""
return plugin_loader.get_all_plugin_configs()
# [/DEF:list_plugins:Function]
# [/DEF:PluginsRouter:Module]
# [/DEF:PluginsRouter:Module]

2
backend/src/api/routes/reports.py
View File

@@ -49,6 +49,7 @@ router = APIRouter(prefix="/api/reports", tags=["Reports"])
# @PARAM: enum_cls (type) - Enum class for validation.
# @PARAM: field_name (str) - Query field name for diagnostics.
# @RETURN: List - Parsed enum values.
# @RELATION: BINDS_TO -> [ReportsRouter]
def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List:
with belief_scope("_parse_csv_enum_list"):
if raw is None or not raw.strip():
@@ -158,6 +159,7 @@ async def list_reports(
# @PURPOSE: Return one normalized report detail with diagnostics and next actions.
# @PRE: authenticated/authorized request and existing report_id.
# @POST: returns normalized detail envelope or 404 when report is not found.
# @RELATION: CALLS -> [ReportsService:Class]
@router.get("/{report_id}", response_model=ReportDetailView)
async def get_report_detail(
report_id: str,

150
backend/src/api/routes/tasks.py
View File

@@ -15,7 +15,12 @@ from ...core.logger import belief_scope
from ...core.task_manager import TaskManager, Task, TaskStatus, LogEntry
from ...core.task_manager.models import LogFilter, LogStats
from ...dependencies import get_task_manager, has_permission, get_current_user, get_config_manager
from ...dependencies import (
get_task_manager,
has_permission,
get_current_user,
get_config_manager,
)
from ...core.config_manager import ConfigManager
from ...services.llm_prompt_templates import (
is_multimodal_model,
@@ -32,16 +37,20 @@ TASK_TYPE_PLUGIN_MAP = {
"migration": ["superset-migration"],
}
class CreateTaskRequest(BaseModel):
plugin_id: str
params: Dict[str, Any]
class ResolveTaskRequest(BaseModel):
resolution_params: Dict[str, Any]
class ResumeTaskRequest(BaseModel):
passwords: Dict[str, str]
# [DEF:create_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Create and start a new task for a given plugin.
@@ -50,11 +59,14 @@ class ResumeTaskRequest(BaseModel):
# @PRE: plugin_id must exist and params must be valid for that plugin.
# @POST: A new task is created and started.
# @RETURN: Task - The created task instance.
# @RELATION: CALLS -> [TaskManager]
# @RELATION: DEPENDS_ON -> [ConfigManager]
# @RELATION: DEPENDS_ON -> [LLMProviderService]
@router.post("", response_model=Task, status_code=status.HTTP_201_CREATED)
async def create_task(
request: CreateTaskRequest,
task_manager: TaskManager = Depends(get_task_manager),
current_user = Depends(get_current_user),
current_user=Depends(get_current_user),
config_manager: ConfigManager = Depends(get_config_manager),
):
# Dynamic permission check based on plugin_id
@@ -65,19 +77,30 @@ async def create_task(
if request.plugin_id in {"llm_dashboard_validation", "llm_documentation"}:
from ...core.database import SessionLocal
from ...services.llm_provider import LLMProviderService
db = SessionLocal()
try:
llm_service = LLMProviderService(db)
provider_id = request.params.get("provider_id")
if not provider_id:
llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm)
binding_key = "dashboard_validation" if request.plugin_id == "llm_dashboard_validation" else "documentation"
provider_id = resolve_bound_provider_id(llm_settings, binding_key)
llm_settings = normalize_llm_settings(
config_manager.get_config().settings.llm
)
binding_key = (
"dashboard_validation"
if request.plugin_id == "llm_dashboard_validation"
else "documentation"
)
provider_id = resolve_bound_provider_id(
llm_settings, binding_key
)
if provider_id:
request.params["provider_id"] = provider_id
if not provider_id:
providers = llm_service.get_all_providers()
active_provider = next((p for p in providers if p.is_active), None)
active_provider = next(
(p for p in providers if p.is_active), None
)
if active_provider:
provider_id = active_provider.id
request.params["provider_id"] = provider_id
@@ -86,9 +109,12 @@ async def create_task(
db_provider = llm_service.get_provider(provider_id)
if not db_provider:
raise ValueError(f"LLM Provider {provider_id} not found")
if request.plugin_id == "llm_dashboard_validation" and not is_multimodal_model(
db_provider.default_model,
db_provider.provider_type,
if (
request.plugin_id == "llm_dashboard_validation"
and not is_multimodal_model(
db_provider.default_model,
db_provider.provider_type,
)
):
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -98,14 +124,16 @@ async def create_task(
db.close()
task = await task_manager.create_task(
plugin_id=request.plugin_id,
params=request.params
plugin_id=request.plugin_id, params=request.params
)
return task
except ValueError as e:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e))
# [/DEF:create_task:Function]
# [DEF:list_tasks:Function]
# @COMPLEXITY: 2
# @PURPOSE: Retrieve a list of tasks with pagination and optional status filter.
@@ -116,16 +144,24 @@ async def create_task(
# @PRE: task_manager must be available.
# @POST: Returns a list of tasks.
# @RETURN: List[Task] - List of tasks.
# @RELATION: CALLS -> [TaskManager]
# @RELATION: BINDS_TO -> [TASK_TYPE_PLUGIN_MAP]
@router.get("", response_model=List[Task])
async def list_tasks(
limit: int = 10,
offset: int = 0,
status_filter: Optional[TaskStatus] = Query(None, alias="status"),
task_type: Optional[str] = Query(None, description="Task category: llm_validation, backup, migration"),
plugin_id: Optional[List[str]] = Query(None, description="Filter by plugin_id (repeatable query param)"),
completed_only: bool = Query(False, description="Return only completed tasks (SUCCESS/FAILED)"),
task_type: Optional[str] = Query(
None, description="Task category: llm_validation, backup, migration"
),
plugin_id: Optional[List[str]] = Query(
None, description="Filter by plugin_id (repeatable query param)"
),
completed_only: bool = Query(
False, description="Return only completed tasks (SUCCESS/FAILED)"
),
task_manager: TaskManager = Depends(get_task_manager),
_ = Depends(has_permission("tasks", "READ"))
_=Depends(has_permission("tasks", "READ")),
):
with belief_scope("list_tasks"):
plugin_filters = list(plugin_id) if plugin_id else []
@@ -133,7 +169,7 @@ async def list_tasks(
if task_type not in TASK_TYPE_PLUGIN_MAP:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Unsupported task_type '{task_type}'. Allowed: {', '.join(TASK_TYPE_PLUGIN_MAP.keys())}"
detail=f"Unsupported task_type '{task_type}'. Allowed: {', '.join(TASK_TYPE_PLUGIN_MAP.keys())}",
)
plugin_filters.extend(TASK_TYPE_PLUGIN_MAP[task_type])
@@ -142,10 +178,13 @@ async def list_tasks(
offset=offset,
status=status_filter,
plugin_ids=plugin_filters or None,
completed_only=completed_only
completed_only=completed_only,
)
# [/DEF:list_tasks:Function]
# [DEF:get_task:Function]
# @COMPLEXITY: 2
# @PURPOSE: Retrieve the details of a specific task.
@@ -154,19 +193,25 @@ async def list_tasks(
# @PRE: task_id must exist.
# @POST: Returns task details or raises 404.
# @RETURN: Task - The task details.
# @RELATION: CALLS -> [TaskManager]
@router.get("/{task_id}", response_model=Task)
async def get_task(
task_id: str,
task_manager: TaskManager = Depends(get_task_manager),
_ = Depends(has_permission("tasks", "READ"))
_=Depends(has_permission("tasks", "READ")),
):
with belief_scope("get_task"):
task = task_manager.get_task(task_id)
if not task:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Task not found"
)
return task
# [/DEF:get_task:Function]
# [DEF:get_task_logs:Function]
# @COMPLEXITY: 5
# @PURPOSE: Retrieve logs for a specific task with optional filtering.
@@ -180,6 +225,8 @@ async def get_task(
# @PRE: task_id must exist.
# @POST: Returns a list of log entries or raises 404.
# @RETURN: List[LogEntry] - List of log entries.
# @RELATION: CALLS -> [TaskManager]
# @RELATION: DEPENDS_ON -> [LogFilter]
# @TEST_CONTRACT: TaskLogQueryInput -> List[LogEntry]
# @TEST_SCENARIO: existing_task_logs_filtered -> Returns filtered logs by level/source/search with pagination.
# @TEST_FIXTURE: valid_task_with_mixed_logs -> backend/tests/fixtures/task_logs/valid_task_with_mixed_logs.json
@@ -190,28 +237,37 @@ async def get_task(
@router.get("/{task_id}/logs")
async def get_task_logs(
task_id: str,
level: Optional[str] = Query(None, description="Filter by log level (DEBUG, INFO, WARNING, ERROR)"),
level: Optional[str] = Query(
None, description="Filter by log level (DEBUG, INFO, WARNING, ERROR)"
),
source: Optional[str] = Query(None, description="Filter by source component"),
search: Optional[str] = Query(None, description="Text search in message"),
offset: int = Query(0, ge=0, description="Number of logs to skip"),
limit: int = Query(100, ge=1, le=1000, description="Maximum number of logs to return"),
limit: int = Query(
100, ge=1, le=1000, description="Maximum number of logs to return"
),
task_manager: TaskManager = Depends(get_task_manager),
):
with belief_scope("get_task_logs"):
task = task_manager.get_task(task_id)
if not task:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Task not found"
)
log_filter = LogFilter(
level=level.upper() if level else None,
source=source,
search=search,
offset=offset,
limit=limit
limit=limit,
)
return task_manager.get_task_logs(task_id, log_filter)
# [/DEF:get_task_logs:Function]
# [DEF:get_task_log_stats:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get statistics about logs for a task (counts by level and source).
@@ -220,6 +276,8 @@ async def get_task_logs(
# @PRE: task_id must exist.
# @POST: Returns log statistics or raises 404.
# @RETURN: LogStats - Statistics about task logs.
# @RELATION: CALLS -> [TaskManager]
# @RELATION: DEPENDS_ON -> [LogStats]
@router.get("/{task_id}/logs/stats", response_model=LogStats)
async def get_task_log_stats(
task_id: str,
@@ -228,26 +286,37 @@ async def get_task_log_stats(
with belief_scope("get_task_log_stats"):
task = task_manager.get_task(task_id)
if not task:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Task not found"
)
stats_payload = task_manager.get_task_log_stats(task_id)
if isinstance(stats_payload, LogStats):
return stats_payload
if isinstance(stats_payload, dict) and (
"total_count" in stats_payload or "by_level" in stats_payload or "by_source" in stats_payload
"total_count" in stats_payload
or "by_level" in stats_payload
or "by_source" in stats_payload
):
return LogStats(
total_count=int(stats_payload.get("total_count", 0) or 0),
by_level=dict(stats_payload.get("by_level") or {}),
by_source=dict(stats_payload.get("by_source") or {}),
)
flat_by_level = dict(stats_payload or {}) if isinstance(stats_payload, dict) else {}
flat_by_level = (
dict(stats_payload or {}) if isinstance(stats_payload, dict) else {}
)
return LogStats(
total_count=sum(int(value or 0) for value in flat_by_level.values()),
by_level={str(key): int(value or 0) for key, value in flat_by_level.items()},
by_level={
str(key): int(value or 0) for key, value in flat_by_level.items()
},
by_source={},
)
# [/DEF:get_task_log_stats:Function]
# [DEF:get_task_log_sources:Function]
# @COMPLEXITY: 2
# @PURPOSE: Get unique sources for a task's logs.
@@ -256,6 +325,7 @@ async def get_task_log_stats(
# @PRE: task_id must exist.
# @POST: Returns list of unique source names or raises 404.
# @RETURN: List[str] - Unique source names.
# @RELATION: CALLS -> [TaskManager]
@router.get("/{task_id}/logs/sources", response_model=List[str])
async def get_task_log_sources(
task_id: str,
@@ -264,10 +334,15 @@ async def get_task_log_sources(
with belief_scope("get_task_log_sources"):
task = task_manager.get_task(task_id)
if not task:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Task not found"
)
return task_manager.get_task_log_sources(task_id)
# [/DEF:get_task_log_sources:Function]
# [DEF:resolve_task:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resolve a task that is awaiting mapping.
@@ -277,12 +352,13 @@ async def get_task_log_sources(
# @PRE: task must be in AWAITING_MAPPING status.
# @POST: Task is resolved and resumes execution.
# @RETURN: Task - The updated task object.
# @RELATION: CALLS -> [TaskManager]
@router.post("/{task_id}/resolve", response_model=Task)
async def resolve_task(
task_id: str,
request: ResolveTaskRequest,
task_manager: TaskManager = Depends(get_task_manager),
_ = Depends(has_permission("tasks", "WRITE"))
_=Depends(has_permission("tasks", "WRITE")),
):
with belief_scope("resolve_task"):
try:
@@ -290,8 +366,11 @@ async def resolve_task(
return task_manager.get_task(task_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:resolve_task:Function]
# [DEF:resume_task:Function]
# @COMPLEXITY: 2
# @PURPOSE: Resume a task that is awaiting input (e.g., passwords).
@@ -301,12 +380,13 @@ async def resolve_task(
# @PRE: task must be in AWAITING_INPUT status.
# @POST: Task resumes execution with provided input.
# @RETURN: Task - The updated task object.
# @RELATION: CALLS -> [TaskManager]
@router.post("/{task_id}/resume", response_model=Task)
async def resume_task(
task_id: str,
request: ResumeTaskRequest,
task_manager: TaskManager = Depends(get_task_manager),
_ = Depends(has_permission("tasks", "WRITE"))
_=Depends(has_permission("tasks", "WRITE")),
):
with belief_scope("resume_task"):
try:
@@ -314,8 +394,11 @@ async def resume_task(
return task_manager.get_task(task_id)
except ValueError as e:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# [/DEF:resume_task:Function]
# [DEF:clear_tasks:Function]
# @COMPLEXITY: 2
# @PURPOSE: Clear tasks matching the status filter.
@@ -323,15 +406,18 @@ async def resume_task(
# @PARAM: task_manager (TaskManager) - The task manager instance.
# @PRE: task_manager is available.
# @POST: Tasks are removed from memory/persistence.
# @RELATION: CALLS -> [TaskManager]
@router.delete("", status_code=status.HTTP_204_NO_CONTENT)
async def clear_tasks(
status: Optional[TaskStatus] = None,
task_manager: TaskManager = Depends(get_task_manager),
_ = Depends(has_permission("tasks", "WRITE"))
_=Depends(has_permission("tasks", "WRITE")),
):
with belief_scope("clear_tasks", f"status={status}"):
task_manager.clear_tasks(status)
return
# [/DEF:clear_tasks:Function]
# [/DEF:TasksRouter:Module]

133
backend/src/app.py
View File

@@ -32,19 +32,43 @@ from .core.logger import logger, belief_scope
from .core.database import AuthSessionLocal
from .core.auth.security import get_password_hash
from .models.auth import User, Role
from .api.routes import plugins, tasks, settings, environments, mappings, migration, connections, git, storage, admin, llm, dashboards, datasets, reports, assistant, clean_release, clean_release_v2, profile, health, dataset_review
from .api.routes import (
plugins,
tasks,
settings,
environments,
mappings,
migration,
connections,
git,
storage,
admin,
llm,
dashboards,
datasets,
reports,
assistant,
clean_release,
clean_release_v2,
profile,
health,
dataset_review,
)
from .api import auth
# [DEF:App:Global]
# @COMPLEXITY: 1
# @SEMANTICS: app, fastapi, instance
# @PURPOSE: The global FastAPI application instance.
# [DEF:FastAPI_App:Global]
# @COMPLEXITY: 3
# @SEMANTICS: app, fastapi, instance, route-registry
# @PURPOSE: Canonical FastAPI application instance for route, middleware, and websocket registration.
# @RELATION: DEPENDS_ON -> [ApiRoutesModule]
# @RELATION: BINDS_TO -> [API_Routes]
app = FastAPI(
title="Superset Tools API",
description="API for managing Superset automation tools and plugins.",
version="1.0.0",
)
# [/DEF:App:Global]
# [/DEF:FastAPI_App:Global]
# [DEF:ensure_initial_admin_user:Function]
# @COMPLEXITY: 3
@@ -72,7 +96,9 @@ def ensure_initial_admin_user() -> None:
existing_user = db.query(User).filter(User.username == username).first()
if existing_user:
logger.info("Initial admin bootstrap skipped: user '%s' already exists.", username)
logger.info(
"Initial admin bootstrap skipped: user '%s' already exists.", username
)
return
new_user = User(
@@ -85,15 +111,20 @@ def ensure_initial_admin_user() -> None:
new_user.roles.append(admin_role)
db.add(new_user)
db.commit()
logger.info("Initial admin user '%s' created from environment bootstrap.", username)
logger.info(
"Initial admin user '%s' created from environment bootstrap.", username
)
except Exception as exc:
db.rollback()
logger.error("Failed to bootstrap initial admin user: %s", exc)
raise
finally:
db.close()
# [/DEF:ensure_initial_admin_user:Function]
# [DEF:startup_event:Function]
# @COMPLEXITY: 3
# @PURPOSE: Handles application startup tasks, such as starting the scheduler.
@@ -108,8 +139,11 @@ async def startup_event():
ensure_initial_admin_user()
scheduler = get_scheduler_service()
scheduler.start()
# [/DEF:startup_event:Function]
# [DEF:shutdown_event:Function]
# @COMPLEXITY: 3
# @PURPOSE: Handles application shutdown tasks, such as stopping the scheduler.
@@ -122,12 +156,15 @@ async def shutdown_event():
with belief_scope("shutdown_event"):
scheduler = get_scheduler_service()
scheduler.stop()
# [/DEF:shutdown_event:Function]
# [DEF:app_middleware:Block]
# @PURPOSE: Configure application-wide middleware (Session, CORS).
# Configure Session Middleware (required by Authlib for OAuth2 flow)
from .core.auth.config import auth_config
app.add_middleware(SessionMiddleware, secret_key=auth_config.SECRET_KEY)
# Configure CORS
@@ -154,10 +191,13 @@ async def network_error_handler(request: Request, exc: NetworkError):
logger.error(f"Network error: {exc}")
return HTTPException(
status_code=503,
detail="Environment unavailable. Please check if the Superset instance is running."
detail="Environment unavailable. Please check if the Superset instance is running.",
)
# [/DEF:network_error_handler:Function]
# [DEF:log_requests:Function]
# @COMPLEXITY: 3
# @PURPOSE: Middleware to log incoming HTTP requests and their response status.
@@ -171,32 +211,50 @@ async def log_requests(request: Request, call_next):
with belief_scope("log_requests"):
# Avoid spamming logs for polling endpoints
is_polling = request.url.path.endswith("/api/tasks") and request.method == "GET"
if not is_polling:
logger.info(f"Incoming request: {request.method} {request.url.path}")
try:
response = await call_next(request)
if not is_polling:
logger.info(f"Response status: {response.status_code} for {request.url.path}")
logger.info(
f"Response status: {response.status_code} for {request.url.path}"
)
return response
except NetworkError as e:
logger.error(f"Network error caught in middleware: {e}")
raise HTTPException(
status_code=503,
detail="Environment unavailable. Please check if the Superset instance is running."
detail="Environment unavailable. Please check if the Superset instance is running.",
)
# [/DEF:log_requests:Function]
# [DEF:api_routes:Block]
# @PURPOSE: Register all application API routers.
# [DEF:API_Routes:Block]
# @COMPLEXITY: 3
# @PURPOSE: Register all FastAPI route groups exposed by the application entrypoint.
# @RELATION: DEPENDS_ON -> [FastAPI_App]
# @RELATION: DEPENDS_ON -> [Route_Group_Contracts]
# @RELATION: DEPENDS_ON -> [AuthApi]
# @RELATION: DEPENDS_ON -> [AdminApi]
# @RELATION: DEPENDS_ON -> [PluginsRouter]
# @RELATION: DEPENDS_ON -> [TasksRouter]
# @RELATION: DEPENDS_ON -> [SettingsRouter]
# @RELATION: DEPENDS_ON -> [ConnectionsRouter]
# @RELATION: DEPENDS_ON -> [ReportsRouter]
# @RELATION: DEPENDS_ON -> [LlmRoutes]
# @RELATION: DEPENDS_ON -> [CleanReleaseV2Api]
# Include API routes
app.include_router(auth.router)
app.include_router(admin.router)
app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"])
app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"])
app.include_router(settings.router, prefix="/api/settings", tags=["Settings"])
app.include_router(connections.router, prefix="/api/settings/connections", tags=["Connections"])
app.include_router(
connections.router, prefix="/api/settings/connections", tags=["Connections"]
)
app.include_router(environments.router, tags=["Environments"])
app.include_router(mappings.router, prefix="/api/mappings", tags=["Mappings"])
app.include_router(migration.router)
@@ -212,7 +270,7 @@ app.include_router(clean_release_v2.router)
app.include_router(profile.router)
app.include_router(dataset_review.router)
app.include_router(health.router)
# [/DEF:api_routes:Block]
# [/DEF:API_Routes:Block]
# [DEF:api.include_routers:Action]
@@ -222,6 +280,7 @@ app.include_router(health.router)
# @SEMANTICS: routes, registration, api
# [/DEF:api.include_routers:Action]
# [DEF:websocket_endpoint:Function]
# @COMPLEXITY: 5
# @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering.
@@ -250,14 +309,11 @@ app.include_router(health.router)
# @TEST_INVARIANT: consistent_streaming -> verifies: [valid_ws_connection]
@app.websocket("/ws/logs/{task_id}")
async def websocket_endpoint(
websocket: WebSocket,
task_id: str,
source: str = None,
level: str = None
websocket: WebSocket, task_id: str, source: str = None, level: str = None
):
"""
WebSocket endpoint for real-time log streaming with optional server-side filtering.
Query Parameters:
source: Filter logs by source component (e.g., "plugin", "superset_api")
level: Filter logs by minimum level (DEBUG, INFO, WARNING, ERROR)
@@ -327,7 +383,9 @@ async def websocket_endpoint(
task = task_manager.get_task(task_id)
if task and task.status == "AWAITING_INPUT" and task.input_request:
synthetic_log = {
"timestamp": task.logs[-1].timestamp.isoformat() if task.logs else "2024-01-01T00:00:00",
"timestamp": task.logs[-1].timestamp.isoformat()
if task.logs
else "2024-01-01T00:00:00",
"level": "INFO",
"message": "Task paused for user input (Connection Re-established)",
"context": {"input_request": task.input_request},
@@ -355,7 +413,10 @@ async def websocket_endpoint(
},
)
if "Task completed successfully" in log_entry.message or "Task failed" in log_entry.message:
if (
"Task completed successfully" in log_entry.message
or "Task failed" in log_entry.message
):
logger.reason(
"Observed terminal task log entry; delaying to preserve client visibility",
extra={"task_id": task_id, "message": log_entry.message},
@@ -379,6 +440,8 @@ async def websocket_endpoint(
"Released WebSocket log queue subscription",
extra={"task_id": task_id},
)
# [/DEF:websocket_endpoint:Function]
# [DEF:StaticFiles:Mount]
@@ -387,7 +450,9 @@ async def websocket_endpoint(
# @PURPOSE: Mounts the frontend build directory to serve static assets.
frontend_path = project_root / "frontend" / "build"
if frontend_path.exists():
app.mount("/_app", StaticFiles(directory=str(frontend_path / "_app")), name="static")
app.mount(
"/_app", StaticFiles(directory=str(frontend_path / "_app")), name="static"
)
# [DEF:serve_spa:Function]
# @COMPLEXITY: 1
@@ -399,15 +464,22 @@ if frontend_path.exists():
with belief_scope("serve_spa"):
# Only serve SPA for non-API paths
# API routes are registered separately and should be matched by FastAPI first
if file_path and (file_path.startswith("api/") or file_path.startswith("/api/") or file_path == "api"):
if file_path and (
file_path.startswith("api/")
or file_path.startswith("/api/")
or file_path == "api"
):
# This should not happen if API routers are properly registered
# Return 404 instead of serving HTML
raise HTTPException(status_code=404, detail=f"API endpoint not found: {file_path}")
raise HTTPException(
status_code=404, detail=f"API endpoint not found: {file_path}"
)
full_path = frontend_path / file_path
if file_path and full_path.is_file():
return FileResponse(str(full_path))
return FileResponse(str(frontend_path / "index.html"))
# [/DEF:serve_spa:Function]
else:
# [DEF:read_root:Function]
@@ -418,7 +490,10 @@ else:
@app.get("/")
async def read_root():
with belief_scope("read_root"):
return {"message": "Superset Tools API is running (Frontend build not found)"}
return {
"message": "Superset Tools API is running (Frontend build not found)"
}
# [/DEF:read_root:Function]
# [/DEF:StaticFiles:Mount]
# [/DEF:AppModule:Module]

36
backend/src/core/config_models.py
View File

@@ -1,10 +1,10 @@
# [DEF:backend.src.core.config_models:Module]
# [DEF:ConfigModels:Module]
# @COMPLEXITY: 3
# @SEMANTICS: config, models, pydantic
# @PURPOSE: Defines the data models for application configuration using Pydantic.
# @LAYER: Core
# @RELATION: READS_FROM -> app_configurations (database)
# @RELATION: USED_BY -> ConfigManager
# @RELATION: IMPLEMENTS -> [CoreContracts]
# @RELATION: IMPLEMENTS -> [ConnectionContracts]
from pydantic import BaseModel, Field
from typing import List, Optional
@@ -15,14 +15,18 @@ from ..services.llm_prompt_templates import (
DEFAULT_LLM_PROVIDER_BINDINGS,
)
# [DEF:Schedule:DataClass]
# @PURPOSE: Represents a backup schedule configuration.
class Schedule(BaseModel):
enabled: bool = False
cron_expression: str = "0 0 * * *" # Default: daily at midnight
# [/DEF:Schedule:DataClass]
# [DEF:backend.src.core.config_models.Environment:DataClass]
# [DEF:Environment:DataClass]
# @PURPOSE: Represents a Superset environment configuration.
class Environment(BaseModel):
id: str
@@ -36,26 +40,37 @@ class Environment(BaseModel):
is_default: bool = False
is_production: bool = False
backup_schedule: Schedule = Field(default_factory=Schedule)
# [/DEF:backend.src.core.config_models.Environment:DataClass]
# [/DEF:Environment:DataClass]
# [DEF:LoggingConfig:DataClass]
# @PURPOSE: Defines the configuration for the application's logging system.
class LoggingConfig(BaseModel):
level: str = "INFO"
task_log_level: str = "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR)
task_log_level: str = (
"INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR)
)
file_path: Optional[str] = None
max_bytes: int = 10 * 1024 * 1024
backup_count: int = 5
enable_belief_state: bool = True
# [/DEF:LoggingConfig:DataClass]
# [DEF:CleanReleaseConfig:DataClass]
# @PURPOSE: Configuration for clean release compliance subsystem.
class CleanReleaseConfig(BaseModel):
active_policy_id: Optional[str] = None
active_registry_id: Optional[str] = None
# [/DEF:CleanReleaseConfig:DataClass]
# [DEF:GlobalSettings:DataClass]
# @PURPOSE: Represents global application settings.
class GlobalSettings(BaseModel):
@@ -73,12 +88,12 @@ class GlobalSettings(BaseModel):
**dict(DEFAULT_LLM_ASSISTANT_SETTINGS),
}
)
# Task retention settings
task_retention_days: int = 30
task_retention_limit: int = 100
pagination_limit: int = 10
# Migration sync settings
migration_sync_cron: str = "0 2 * * *"
@@ -86,13 +101,18 @@ class GlobalSettings(BaseModel):
ff_dataset_auto_review: bool = True
ff_dataset_clarification: bool = True
ff_dataset_execution: bool = True
# [/DEF:GlobalSettings:DataClass]
# [DEF:AppConfig:DataClass]
# @PURPOSE: The root configuration model containing all application settings.
class AppConfig(BaseModel):
environments: List[Environment] = []
settings: GlobalSettings
# [/DEF:AppConfig:DataClass]
# [/DEF:ConfigModels:Module]

282
backend/src/core/superset_client.py
View File

@@ -4,7 +4,8 @@
# @SEMANTICS: superset, api, client, rest, http, dashboard, dataset, import, export
# @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию.
# @LAYER: Core
# @RELATION: [DEPENDS_ON] ->[APIClient.__init__]
# @RELATION: DEPENDS_ON -> [ConfigModels]
# @RELATION: DEPENDS_ON -> [ConnectionContracts]
#
# @INVARIANT: All network operations must use the internal APIClient instance.
# @PUBLIC_API: SupersetClient
@@ -30,16 +31,17 @@ app_logger = cast(Any, app_logger)
# [DEF:SupersetClient:Class]
# @COMPLEXITY: 3
# @PURPOSE: Класс-обёртка над Superset REST API, предоставляющий методы для работы с дашбордами и датасетами.
# @RELATION: [DEPENDS_ON] ->[APIClient]
# @RELATION: DEPENDS_ON -> [ConfigModels]
# @RELATION: DEPENDS_ON -> [ConnectionContracts]
class SupersetClient:
# [DEF:SupersetClient.__init__:Function]
# [DEF:SupersetClientInit:Function]
# @COMPLEXITY: 3
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
# @PRE: `env` должен быть валидным объектом Environment.
# @POST: Атрибуты `env` и `network` созданы и готовы к работе.
# @DATA_CONTRACT: Input[Environment] -> self.network[APIClient]
# @RELATION: [DEPENDS_ON] ->[Environment]
# @RELATION: [DEPENDS_ON] ->[APIClient.__init__]
# @RELATION: DEPENDS_ON -> [Environment]
# @RELATION: DEPENDS_ON -> [ConnectionContracts]
def __init__(self, env: Environment):
with belief_scope("__init__"):
app_logger.info(
@@ -62,23 +64,23 @@ class SupersetClient:
self.delete_before_reimport: bool = False
app_logger.info("[SupersetClient.__init__][Exit] SupersetClient initialized.")
# [/DEF:SupersetClient.__init__:Function]
# [/DEF:SupersetClientInit:Function]
# [DEF:SupersetClient.authenticate:Function]
# [DEF:SupersetClientAuthenticate:Function]
# @COMPLEXITY: 3
# @PURPOSE: Authenticates the client using the configured credentials.
# @PRE: self.network must be initialized with valid auth configuration.
# @POST: Client is authenticated and tokens are stored.
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
# @RELATION: [CALLS] ->[APIClient.authenticate]
# @RELATION: CALLS -> [ConnectionContracts]
def authenticate(self) -> Dict[str, str]:
with belief_scope("SupersetClient.authenticate"):
return self.network.authenticate()
# [/DEF:SupersetClient.authenticate:Function]
# [/DEF:SupersetClientAuthenticate:Function]
@property
# [DEF:SupersetClient.headers:Function]
# [DEF:SupersetClientHeaders:Function]
# @COMPLEXITY: 1
# @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
# @PRE: APIClient is initialized and authenticated.
@@ -87,17 +89,17 @@ class SupersetClient:
with belief_scope("headers"):
return self.network.headers
# [/DEF:SupersetClient.headers:Function]
# [/DEF:SupersetClientHeaders:Function]
# [SECTION: DASHBOARD OPERATIONS]
# [DEF:SupersetClient.get_dashboards:Function]
# [DEF:SupersetClientGetDashboards:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.
# @PRE: Client is authenticated.
# @POST: Returns a tuple with total count and list of dashboards.
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
# @RELATION: [CALLS] ->[_fetch_all_pages]
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_dashboards"):
app_logger.info("[get_dashboards][Enter] Fetching dashboards.")
@@ -127,15 +129,15 @@ class SupersetClient:
app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count)
return total_count, paginated_data
# [/DEF:SupersetClient.get_dashboards:Function]
# [/DEF:SupersetClientGetDashboards:Function]
# [DEF:SupersetClient.get_dashboards_page:Function]
# [DEF:SupersetClientGetDashboardsPage:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.
# @PRE: Client is authenticated.
# @POST: Returns total count and one page of dashboards.
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def get_dashboards_page(
self, query: Optional[Dict] = None
) -> Tuple[int, List[Dict]]:
@@ -167,15 +169,15 @@ class SupersetClient:
total_count = response_json.get("count", len(result))
return total_count, result
# [/DEF:SupersetClient.get_dashboards_page:Function]
# [/DEF:SupersetClientGetDashboardsPage:Function]
# [DEF:SupersetClient.get_dashboards_summary:Function]
# [DEF:SupersetClientGetDashboardsSummary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches dashboard metadata optimized for the grid.
# @PRE: Client is authenticated.
# @POST: Returns a list of dashboard metadata summaries.
# @DATA_CONTRACT: None -> Output[List[Dict]]
# @RELATION: [CALLS] ->[SupersetClient.get_dashboards]
# @RELATION: CALLS -> [SupersetClientGetDashboards]
def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:
with belief_scope("SupersetClient.get_dashboards_summary"):
# Rely on list endpoint default projection to stay compatible
@@ -257,15 +259,15 @@ class SupersetClient:
)
return result
# [/DEF:SupersetClient.get_dashboards_summary:Function]
# [/DEF:SupersetClientGetDashboardsSummary:Function]
# [DEF:SupersetClient.get_dashboards_summary_page:Function]
# [DEF:SupersetClientGetDashboardsSummaryPage:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.
# @PRE: page >= 1 and page_size > 0.
# @POST: Returns mapped summaries and total dashboard count.
# @DATA_CONTRACT: Input[page: int, page_size: int] -> Output[Tuple[int, List[Dict]]]
# @RELATION: [CALLS] ->[SupersetClient.get_dashboards_page]
# @RELATION: CALLS -> [SupersetClientGetDashboardsPage]
def get_dashboards_summary_page(
self,
page: int,
@@ -333,9 +335,9 @@ class SupersetClient:
return total_count, result
# [/DEF:SupersetClient.get_dashboards_summary_page:Function]
# [/DEF:SupersetClientGetDashboardsSummaryPage:Function]
# [DEF:SupersetClient._extract_owner_labels:Function]
# [DEF:SupersetClientExtractOwnerLabels:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize dashboard owners payload to stable display labels.
# @PRE: owners payload can be scalar, object or list.
@@ -362,9 +364,9 @@ class SupersetClient:
normalized.append(label)
return normalized
# [/DEF:SupersetClient._extract_owner_labels:Function]
# [/DEF:SupersetClientExtractOwnerLabels:Function]
# [DEF:SupersetClient._extract_user_display:Function]
# [DEF:SupersetClientExtractUserDisplay:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize user payload to a stable display name.
# @PRE: user payload can be string, dict or None.
@@ -396,9 +398,9 @@ class SupersetClient:
return email
return None
# [/DEF:SupersetClient._extract_user_display:Function]
# [/DEF:SupersetClientExtractUserDisplay:Function]
# [DEF:SupersetClient._sanitize_user_text:Function]
# [DEF:SupersetClientSanitizeUserText:Function]
# @COMPLEXITY: 1
# @PURPOSE: Convert scalar value to non-empty user-facing text.
# @PRE: value can be any scalar type.
@@ -411,15 +413,15 @@ class SupersetClient:
return None
return normalized
# [/DEF:SupersetClient._sanitize_user_text:Function]
# [/DEF:SupersetClientSanitizeUserText:Function]
# [DEF:SupersetClient.get_dashboard:Function]
# [DEF:SupersetClientGetDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single dashboard by ID or slug.
# @PRE: Client is authenticated and dashboard_ref exists.
# @POST: Returns dashboard payload from Superset API.
# @DATA_CONTRACT: Input[dashboard_ref: Union[int, str]] -> Output[Dict]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict:
with belief_scope("SupersetClient.get_dashboard", f"ref={dashboard_ref}"):
response = self.network.request(
@@ -427,15 +429,15 @@ class SupersetClient:
)
return cast(Dict, response)
# [/DEF:SupersetClient.get_dashboard:Function]
# [/DEF:SupersetClientGetDashboard:Function]
# [DEF:SupersetClient.get_dashboard_permalink_state:Function]
# [DEF:SupersetClientGetDashboardPermalinkState:Function]
# @COMPLEXITY: 2
# @PURPOSE: Fetches stored dashboard permalink state by permalink key.
# @PRE: Client is authenticated and permalink key exists.
# @POST: Returns dashboard permalink state payload from Superset API.
# @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def get_dashboard_permalink_state(self, permalink_key: str) -> Dict:
with belief_scope(
"SupersetClient.get_dashboard_permalink_state", f"key={permalink_key}"
@@ -445,15 +447,15 @@ class SupersetClient:
)
return cast(Dict, response)
# [/DEF:SupersetClient.get_dashboard_permalink_state:Function]
# [/DEF:SupersetClientGetDashboardPermalinkState:Function]
# [DEF:SupersetClient.get_native_filter_state:Function]
# [DEF:SupersetClientGetNativeFilterState:Function]
# @COMPLEXITY: 2
# @PURPOSE: Fetches stored native filter state by filter state key.
# @PRE: Client is authenticated and filter_state_key exists.
# @POST: Returns native filter state payload from Superset API.
# @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def get_native_filter_state(
self, dashboard_id: Union[int, str], filter_state_key: str
) -> Dict:
@@ -467,15 +469,15 @@ class SupersetClient:
)
return cast(Dict, response)
# [/DEF:SupersetClient.get_native_filter_state:Function]
# [/DEF:SupersetClientGetNativeFilterState:Function]
# [DEF:SupersetClient.extract_native_filters_from_permalink:Function]
# [DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract native filters dataMask from a permalink key.
# @PRE: Client is authenticated and permalink_key exists.
# @POST: Returns extracted dataMask with filter states.
# @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict]
# @RELATION: [CALLS] ->[SupersetClient.get_dashboard_permalink_state]
# @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState]
def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict:
with belief_scope(
"SupersetClient.extract_native_filters_from_permalink",
@@ -507,15 +509,15 @@ class SupersetClient:
"permalink_key": permalink_key,
}
# [/DEF:SupersetClient.extract_native_filters_from_permalink:Function]
# [/DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]
# [DEF:SupersetClient.extract_native_filters_from_key:Function]
# [DEF:SupersetClientExtractNativeFiltersFromKey:Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract native filters from a native_filters_key URL parameter.
# @PRE: Client is authenticated, dashboard_id and filter_state_key exist.
# @POST: Returns extracted filter state with extraFormData.
# @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict]
# @RELATION: [CALLS] ->[SupersetClient.get_native_filter_state]
# @RELATION: CALLS -> [SupersetClientGetNativeFilterState]
def extract_native_filters_from_key(
self, dashboard_id: Union[int, str], filter_state_key: str
) -> Dict:
@@ -576,16 +578,16 @@ class SupersetClient:
"filter_state_key": filter_state_key,
}
# [/DEF:SupersetClient.extract_native_filters_from_key:Function]
# [/DEF:SupersetClientExtractNativeFiltersFromKey:Function]
# [DEF:SupersetClient.parse_dashboard_url_for_filters:Function]
# [DEF:SupersetClientParseDashboardUrlForFilters:Function]
# @COMPLEXITY: 3
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present.
# @PRE: url must be a valid Superset dashboard URL with optional permalink or native_filters_key.
# @POST: Returns extracted filter state or empty dict if no filters found.
# @DATA_CONTRACT: Input[url: str] -> Output[Dict]
# @RELATION: [CALLS] ->[SupersetClient.extract_native_filters_from_permalink]
# @RELATION: [CALLS] ->[SupersetClient.extract_native_filters_from_key]
# @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]
# @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey]
def parse_dashboard_url_for_filters(self, url: str) -> Dict:
with belief_scope(
"SupersetClient.parse_dashboard_url_for_filters", f"url={url}"
@@ -686,30 +688,30 @@ class SupersetClient:
return result
# [/DEF:SupersetClient.parse_dashboard_url_for_filters:Function]
# [/DEF:SupersetClientParseDashboardUrlForFilters:Function]
# [DEF:SupersetClient.get_chart:Function]
# [DEF:SupersetClientGetChart:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single chart by ID.
# @PRE: Client is authenticated and chart_id exists.
# @POST: Returns chart payload from Superset API.
# @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def get_chart(self, chart_id: int) -> Dict:
with belief_scope("SupersetClient.get_chart", f"id={chart_id}"):
response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}")
return cast(Dict, response)
# [/DEF:SupersetClient.get_chart:Function]
# [/DEF:SupersetClientGetChart:Function]
# [DEF:SupersetClient.get_dashboard_detail:Function]
# [DEF:SupersetClientGetDashboardDetail:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches detailed dashboard information including related charts and datasets.
# @PRE: Client is authenticated and dashboard reference exists.
# @POST: Returns dashboard metadata with charts and datasets lists.
# @DATA_CONTRACT: Input[dashboard_ref: Union[int, str]] -> Output[Dict]
# @RELATION: [CALLS] ->[SupersetClient.get_dashboard]
# @RELATION: [CALLS] ->[SupersetClient.get_chart]
# @RELATION: CALLS -> [SupersetClientGetDashboard]
# @RELATION: CALLS -> [SupersetClientGetChart]
def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:
with belief_scope(
"SupersetClient.get_dashboard_detail", f"ref={dashboard_ref}"
@@ -988,15 +990,15 @@ class SupersetClient:
"dataset_count": len(unique_datasets),
}
# [/DEF:SupersetClient.get_dashboard_detail:Function]
# [/DEF:SupersetClientGetDashboardDetail:Function]
# [DEF:SupersetClient.get_charts:Function]
# [DEF:SupersetClientGetCharts:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches all charts with pagination support.
# @PRE: Client is authenticated.
# @POST: Returns total count and charts list.
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
# @RELATION: [CALLS] ->[_fetch_all_pages]
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_charts"):
validated_query = self._validate_query_params(query or {})
@@ -1012,9 +1014,9 @@ class SupersetClient:
)
return len(paginated_data), paginated_data
# [/DEF:SupersetClient.get_charts:Function]
# [/DEF:SupersetClientGetCharts:Function]
# [DEF:SupersetClient._extract_chart_ids_from_layout:Function]
# [DEF:SupersetClientExtractChartIdsFromLayout:Function]
# @COMPLEXITY: 1
# @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys.
# @PRE: payload can be dict/list/scalar.
@@ -1048,16 +1050,16 @@ class SupersetClient:
walk(payload)
return found
# [/DEF:SupersetClient._extract_chart_ids_from_layout:Function]
# [/DEF:SupersetClientExtractChartIdsFromLayout:Function]
# [DEF:export_dashboard:Function]
# [DEF:SupersetClientExportDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Экспортирует дашборд в виде ZIP-архива.
# @PRE: dashboard_id must exist in Superset.
# @POST: Returns ZIP content and filename.
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Tuple[bytes, str]]
# @SIDE_EFFECT: Performs network I/O to download archive.
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:
with belief_scope("export_dashboard"):
app_logger.info(
@@ -1080,17 +1082,17 @@ class SupersetClient:
)
return response.content, filename
# [/DEF:export_dashboard:Function]
# [/DEF:SupersetClientExportDashboard:Function]
# [DEF:import_dashboard:Function]
# [DEF:SupersetClientImportDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Импортирует дашборд из ZIP-файла.
# @PRE: file_name must be a valid ZIP dashboard export.
# @POST: Dashboard is imported or re-imported after deletion.
# @DATA_CONTRACT: Input[file_name: Union[str, Path]] -> Output[Dict]
# @SIDE_EFFECT: Performs network I/O to upload archive.
# @RELATION: [CALLS] ->[SupersetClient._do_import]
# @RELATION: [CALLS] ->[delete_dashboard]
# @RELATION: CALLS -> [SupersetClientDoImport]
# @RELATION: CALLS -> [ConnectionContracts]
def import_dashboard(
self,
file_name: Union[str, Path],
@@ -1127,15 +1129,15 @@ class SupersetClient:
)
return self._do_import(file_path)
# [/DEF:import_dashboard:Function]
# [/DEF:SupersetClientImportDashboard:Function]
# [DEF:delete_dashboard:Function]
# [DEF:SupersetClientDeleteDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Удаляет дашборд по его ID или slug.
# @PRE: dashboard_id must exist.
# @POST: Dashboard is removed from Superset.
# @SIDE_EFFECT: Deletes resource from upstream Superset environment.
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:
with belief_scope("delete_dashboard"):
app_logger.info(
@@ -1156,15 +1158,15 @@ class SupersetClient:
response,
)
# [/DEF:delete_dashboard:Function]
# [/DEF:SupersetClientDeleteDashboard:Function]
# [DEF:SupersetClient.get_datasets:Function]
# [DEF:SupersetClientGetDatasets:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.
# @PRE: Client is authenticated.
# @POST: Returns total count and list of datasets.
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
# @RELATION: [CALLS] ->[_fetch_all_pages]
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_datasets"):
app_logger.info("[get_datasets][Enter] Fetching datasets.")
@@ -1181,15 +1183,15 @@ class SupersetClient:
app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count)
return total_count, paginated_data
# [/DEF:SupersetClient.get_datasets:Function]
# [/DEF:SupersetClientGetDatasets:Function]
# [DEF:SupersetClient.get_datasets_summary:Function]
# [DEF:SupersetClientGetDatasetsSummary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.
# @PRE: Client is authenticated.
# @POST: Returns a list of dataset metadata summaries.
# @RETURN: List[Dict]
# @RELATION: [CALLS] ->[SupersetClient.get_datasets]
# @RELATION: CALLS -> [SupersetClientGetDatasets]
def get_datasets_summary(self) -> List[Dict]:
with belief_scope("SupersetClient.get_datasets_summary"):
query = {"columns": ["id", "table_name", "schema", "database"]}
@@ -1210,17 +1212,17 @@ class SupersetClient:
)
return result
# [/DEF:SupersetClient.get_datasets_summary:Function]
# [/DEF:SupersetClientGetDatasetsSummary:Function]
# [DEF:get_dataset_detail:Function]
# [DEF:SupersetClientGetDatasetDetail:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches detailed dataset information including columns and linked dashboards
# @PRE: Client is authenticated and dataset_id exists.
# @POST: Returns detailed dataset info with columns and linked dashboards.
# @PARAM: dataset_id (int) - The dataset ID to fetch details for.
# @RETURN: Dict - Dataset details with columns and linked_dashboards.
# @RELATION: [CALLS] ->[SupersetClient.get_dataset]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [SupersetClientGetDataset]
# @RELATION: CALLS -> [ConnectionContracts]
def get_dataset_detail(self, dataset_id: int) -> Dict:
with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"):
@@ -1339,15 +1341,15 @@ class SupersetClient:
)
return result
# [/DEF:get_dataset_detail:Function]
# [/DEF:SupersetClientGetDatasetDetail:Function]
# [DEF:SupersetClient.get_dataset:Function]
# [DEF:SupersetClientGetDataset:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает информацию о конкретном датасете по его ID.
# @PRE: dataset_id must exist.
# @POST: Returns dataset details.
# @DATA_CONTRACT: Input[dataset_id: int] -> Output[Dict]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def get_dataset(self, dataset_id: int) -> Dict:
with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"):
app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id)
@@ -1358,19 +1360,19 @@ class SupersetClient:
app_logger.info("[get_dataset][Exit] Got dataset %s.", dataset_id)
return response
# [/DEF:SupersetClient.get_dataset:Function]
# [/DEF:SupersetClientGetDataset:Function]
# [DEF:SupersetClient.compile_dataset_preview:Function]
# [DEF:SupersetClientCompileDatasetPreview:Function]
# @COMPLEXITY: 4
# @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.
# @PRE: dataset_id must be valid and template_params/effective_filters must represent the current preview session inputs.
# @POST: Returns normalized compiled SQL plus raw upstream response, preferring legacy form_data transport with explicit fallback to chart-data.
# @DATA_CONTRACT: Input[dataset_id:int, template_params:Dict, effective_filters:List[Dict]] -> Output[Dict[str, Any]]
# @RELATION: [CALLS] ->[SupersetClient.get_dataset]
# @RELATION: [CALLS] ->[SupersetClient.build_dataset_preview_query_context]
# @RELATION: [CALLS] ->[SupersetClient.build_dataset_preview_legacy_form_data]
# @RELATION: [CALLS] ->[request]
# @RELATION: [CALLS] ->[_extract_compiled_sql_from_preview_response]
# @RELATION: CALLS -> [SupersetClientGetDataset]
# @RELATION: CALLS -> [SupersetClientBuildDatasetPreviewQueryContext]
# @RELATION: CALLS -> [SupersetClientBuildDatasetPreviewLegacyFormData]
# @RELATION: CALLS -> [ConnectionContracts]
# @RELATION: CALLS -> [SupersetClientExtractCompiledSqlFromPreviewResponse]
# @SIDE_EFFECT: Performs upstream dataset lookup and preview network I/O against Superset.
def compile_dataset_preview(
self,
@@ -1535,15 +1537,15 @@ class SupersetClient:
f"(attempts={strategy_attempts!r})"
)
# [/DEF:SupersetClient.compile_dataset_preview:Function]
# [/DEF:SupersetClientCompileDatasetPreview:Function]
# [DEF:SupersetClient.build_dataset_preview_legacy_form_data:Function]
# [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]
# @COMPLEXITY: 4
# @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.
# @PRE: dataset_record should come from Superset dataset detail when possible.
# @POST: Returns one serialized-ready form_data structure preserving native filter clauses in legacy transport fields.
# @DATA_CONTRACT: Input[dataset_id:int,dataset_record:Dict,template_params:Dict,effective_filters:List[Dict]] -> Output[Dict[str, Any]]
# @RELATION: [CALLS] ->[SupersetClient.build_dataset_preview_query_context]
# @RELATION: CALLS -> [SupersetClientBuildDatasetPreviewQueryContext]
# @SIDE_EFFECT: Emits reasoning diagnostics describing the inferred legacy payload shape.
def build_dataset_preview_legacy_form_data(
self,
@@ -1611,15 +1613,15 @@ class SupersetClient:
)
return legacy_form_data
# [/DEF:SupersetClient.build_dataset_preview_legacy_form_data:Function]
# [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]
# [DEF:SupersetClient.build_dataset_preview_query_context:Function]
# [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]
# @COMPLEXITY: 4
# @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.
# @PRE: dataset_record should come from Superset dataset detail when possible.
# @POST: Returns an explicit chart-data payload based on current session inputs and dataset metadata.
# @DATA_CONTRACT: Input[dataset_id:int,dataset_record:Dict,template_params:Dict,effective_filters:List[Dict]] -> Output[Dict[str, Any]]
# @RELATION: [CALLS] ->[_normalize_effective_filters_for_query_context]
# @RELATION: CALLS -> [SupersetClientNormalizeEffectiveFiltersForQueryContext]
# @SIDE_EFFECT: Emits reasoning and reflection logs for deterministic preview payload construction.
def build_dataset_preview_query_context(
self,
@@ -1740,14 +1742,14 @@ class SupersetClient:
)
return payload
# [/DEF:SupersetClient.build_dataset_preview_query_context:Function]
# [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]
# [DEF:_normalize_effective_filters_for_query_context:Function]
# [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
# @COMPLEXITY: 3
# @PURPOSE: Convert execution mappings into Superset chart-data filter objects.
# @PRE: effective_filters may contain mapping metadata and arbitrary scalar/list values.
# @POST: Returns only valid filter dictionaries suitable for the chart-data query payload.
# @RELATION: [DEPENDS_ON] ->[FilterStateModels]
# @RELATION: DEPENDS_ON -> [CoreContracts]
def _normalize_effective_filters_for_query_context(
self,
effective_filters: List[Dict[str, Any]],
@@ -1850,14 +1852,14 @@ class SupersetClient:
"diagnostics": diagnostics,
}
# [/DEF:_normalize_effective_filters_for_query_context:Function]
# [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
# [DEF:_extract_compiled_sql_from_preview_response:Function]
# [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses.
# @PRE: response must be the decoded preview response body from a supported Superset endpoint.
# @POST: Returns compiled SQL and raw response or raises SupersetAPIError when the endpoint does not expose query text.
# @RELATION: [DEPENDS_ON] ->[SupersetAPIError]
# @RELATION: DEPENDS_ON -> [ConnectionContracts]
def _extract_compiled_sql_from_preview_response(
self, response: Any
) -> Dict[str, Any]:
@@ -1930,16 +1932,16 @@ class SupersetClient:
f"(diagnostics={response_diagnostics!r})"
)
# [/DEF:_extract_compiled_sql_from_preview_response:Function]
# [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
# [DEF:SupersetClient.update_dataset:Function]
# [DEF:SupersetClientUpdateDataset:Function]
# @COMPLEXITY: 3
# @PURPOSE: Обновляет данные датасета по его ID.
# @PRE: dataset_id must exist.
# @POST: Dataset is updated in Superset.
# @DATA_CONTRACT: Input[dataset_id: int, data: Dict] -> Output[Dict]
# @SIDE_EFFECT: Modifies resource in upstream Superset environment.
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def update_dataset(self, dataset_id: int, data: Dict) -> Dict:
with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"):
app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id)
@@ -1953,15 +1955,15 @@ class SupersetClient:
app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id)
return response
# [/DEF:SupersetClient.update_dataset:Function]
# [/DEF:SupersetClientUpdateDataset:Function]
# [DEF:SupersetClient.get_databases:Function]
# [DEF:SupersetClientGetDatabases:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список баз данных.
# @PRE: Client is authenticated.
# @POST: Returns total count and list of databases.
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
# @RELATION: [CALLS] ->[_fetch_all_pages]
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_databases"):
app_logger.info("[get_databases][Enter] Fetching databases.")
@@ -1980,15 +1982,15 @@ class SupersetClient:
app_logger.info("[get_databases][Exit] Found %d databases.", total_count)
return total_count, paginated_data
# [/DEF:SupersetClient.get_databases:Function]
# [/DEF:SupersetClientGetDatabases:Function]
# [DEF:get_database:Function]
# [DEF:SupersetClientGetDatabase:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает информацию о конкретной базе данных по её ID.
# @PRE: database_id must exist.
# @POST: Returns database details.
# @DATA_CONTRACT: Input[database_id: int] -> Output[Dict]
# @RELATION: [CALLS] ->[request]
# @RELATION: CALLS -> [ConnectionContracts]
def get_database(self, database_id: int) -> Dict:
with belief_scope("get_database"):
app_logger.info("[get_database][Enter] Fetching database %s.", database_id)
@@ -1999,15 +2001,15 @@ class SupersetClient:
app_logger.info("[get_database][Exit] Got database %s.", database_id)
return response
# [/DEF:get_database:Function]
# [/DEF:SupersetClientGetDatabase:Function]
# [DEF:get_databases_summary:Function]
# [DEF:SupersetClientGetDatabasesSummary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch a summary of databases including uuid, name, and engine.
# @PRE: Client is authenticated.
# @POST: Returns list of database summaries.
# @DATA_CONTRACT: None -> Output[List[Dict]]
# @RELATION: [CALLS] ->[SupersetClient.get_databases]
# @RELATION: CALLS -> [SupersetClientGetDatabases]
def get_databases_summary(self) -> List[Dict]:
with belief_scope("SupersetClient.get_databases_summary"):
query = {"columns": ["uuid", "database_name", "backend"]}
@@ -2019,29 +2021,29 @@ class SupersetClient:
return databases
# [/DEF:get_databases_summary:Function]
# [/DEF:SupersetClientGetDatabasesSummary:Function]
# [DEF:get_database_by_uuid:Function]
# [DEF:SupersetClientGetDatabaseByUuid:Function]
# @COMPLEXITY: 3
# @PURPOSE: Find a database by its UUID.
# @PRE: db_uuid must be a valid UUID string.
# @POST: Returns database info or None.
# @DATA_CONTRACT: Input[db_uuid: str] -> Output[Optional[Dict]]
# @RELATION: [CALLS] ->[SupersetClient.get_databases]
# @RELATION: CALLS -> [SupersetClientGetDatabases]
def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]:
with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"):
query = {"filters": [{"col": "uuid", "op": "eq", "value": db_uuid}]}
_, databases = self.get_databases(query=query)
return databases[0] if databases else None
# [/DEF:get_database_by_uuid:Function]
# [/DEF:SupersetClientGetDatabaseByUuid:Function]
# [DEF:SupersetClient._resolve_target_id_for_delete:Function]
# [DEF:SupersetClientResolveTargetIdForDelete:Function]
# @COMPLEXITY: 1
# @PURPOSE: Resolves a dashboard ID from either an ID or a slug.
# @PRE: Either dash_id or dash_slug should be provided.
# @POST: Returns the resolved ID or None.
# @RELATION: [CALLS] ->[SupersetClient.get_dashboards]
# @RELATION: CALLS -> [SupersetClientGetDashboards]
def _resolve_target_id_for_delete(
self, dash_id: Optional[int], dash_slug: Optional[str]
) -> Optional[int]:
@@ -2074,14 +2076,14 @@ class SupersetClient:
)
return None
# [/DEF:SupersetClient._resolve_target_id_for_delete:Function]
# [/DEF:SupersetClientResolveTargetIdForDelete:Function]
# [DEF:SupersetClient._do_import:Function]
# [DEF:SupersetClientDoImport:Function]
# @COMPLEXITY: 1
# @PURPOSE: Performs the actual multipart upload for import.
# @PRE: file_name must be a path to an existing ZIP file.
# @POST: Returns the API response from the upload.
# @RELATION: [CALLS] ->[APIClient.upload_file]
# @RELATION: CALLS -> [ConnectionContracts]
def _do_import(self, file_name: Union[str, Path]) -> Dict:
with belief_scope("_do_import"):
app_logger.debug(f"[_do_import][State] Uploading file: {file_name}")
@@ -2103,9 +2105,9 @@ class SupersetClient:
timeout=self.env.timeout * 2,
)
# [/DEF:SupersetClient._do_import:Function]
# [/DEF:SupersetClientDoImport:Function]
# [DEF:_validate_export_response:Function]
# [DEF:SupersetClientValidateExportResponse:Function]
# @COMPLEXITY: 1
# @PURPOSE: Validates that the export response is a non-empty ZIP archive.
# @PRE: response must be a valid requests.Response object.
@@ -2120,9 +2122,9 @@ class SupersetClient:
if not response.content:
raise SupersetAPIError("Получены пустые данные при экспорте")
# [/DEF:_validate_export_response:Function]
# [/DEF:SupersetClientValidateExportResponse:Function]
# [DEF:_resolve_export_filename:Function]
# [DEF:SupersetClientResolveExportFilename:Function]
# @COMPLEXITY: 1
# @PURPOSE: Determines the filename for an exported dashboard.
# @PRE: response must contain Content-Disposition header or dashboard_id must be provided.
@@ -2141,9 +2143,9 @@ class SupersetClient:
)
return filename
# [/DEF:_resolve_export_filename:Function]
# [/DEF:SupersetClientResolveExportFilename:Function]
# [DEF:_validate_query_params:Function]
# [DEF:SupersetClientValidateQueryParams:Function]
# @COMPLEXITY: 1
# @PURPOSE: Ensures query parameters have default page and page_size.
# @PRE: query can be None or a dictionary.
@@ -2155,14 +2157,14 @@ class SupersetClient:
base_query = {"page": 0, "page_size": 100}
return {**base_query, **(query or {})}
# [/DEF:_validate_query_params:Function]
# [/DEF:SupersetClientValidateQueryParams:Function]
# [DEF:_fetch_total_object_count:Function]
# [DEF:SupersetClientFetchTotalObjectCount:Function]
# @COMPLEXITY: 1
# @PURPOSE: Fetches the total number of items for a given endpoint.
# @PRE: endpoint must be a valid Superset API path.
# @POST: Returns the total count as an integer.
# @RELATION: [CALLS] ->[fetch_paginated_count]
# @RELATION: CALLS -> [ConnectionContracts]
def _fetch_total_object_count(self, endpoint: str) -> int:
with belief_scope("_fetch_total_object_count"):
return self.network.fetch_paginated_count(
@@ -2171,23 +2173,23 @@ class SupersetClient:
count_field="count",
)
# [/DEF:_fetch_total_object_count:Function]
# [/DEF:SupersetClientFetchTotalObjectCount:Function]
# [DEF:_fetch_all_pages:Function]
# [DEF:SupersetClientFetchAllPages:Function]
# @COMPLEXITY: 1
# @PURPOSE: Iterates through all pages to collect all data items.
# @PRE: pagination_options must contain base_query, total_count, and results_field.
# @POST: Returns a combined list of all items.
# @RELATION: [CALLS] ->[fetch_paginated_data]
# @RELATION: CALLS -> [ConnectionContracts]
def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]:
with belief_scope("_fetch_all_pages"):
return self.network.fetch_paginated_data(
endpoint=endpoint, pagination_options=pagination_options
)
# [/DEF:_fetch_all_pages:Function]
# [/DEF:SupersetClientFetchAllPages:Function]
# [DEF:_validate_import_file:Function]
# [DEF:SupersetClientValidateImportFile:Function]
# @COMPLEXITY: 1
# @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml.
# @PRE: zip_path must be a path to a file.
@@ -2205,16 +2207,16 @@ class SupersetClient:
f"Архив {zip_path} не содержит 'metadata.yaml'"
)
# [/DEF:_validate_import_file:Function]
# [/DEF:SupersetClientValidateImportFile:Function]
# [DEF:get_all_resources:Function]
# [DEF:SupersetClientGetAllResources:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.
# @PARAM: resource_type (str) - One of "chart", "dataset", "dashboard".
# @PRE: Client is authenticated. resource_type is valid.
# @POST: Returns a list of resource dicts with at minimum id, uuid, and name fields.
# @RETURN: List[Dict]
# @RELATION: [CALLS] ->[_fetch_all_pages]
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_all_resources(
self, resource_type: str, since_dttm: Optional[datetime] = None
) -> List[Dict]:
@@ -2268,7 +2270,7 @@ class SupersetClient:
)
return data
# [/DEF:get_all_resources:Function]
# [/DEF:SupersetClientGetAllResources:Function]
# [/DEF:SupersetClient:Class]

4
backend/src/core/task_manager/__init__.py
View File

@@ -5,7 +5,7 @@
# @LAYER: Core
# @RELATION: DEPENDS_ON ->[TaskManagerModels]
# @RELATION: DEPENDS_ON ->[TaskManagerModule]
# @RELATION: DEPENDS_ON ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: DEPENDS_ON ->[TaskManager]
# @INVARIANT: Package exports stay aligned with manager and models contracts.
from .models import Task, TaskStatus, LogEntry
@@ -13,4 +13,4 @@ from .manager import TaskManager
__all__ = ["TaskManager", "Task", "TaskStatus", "LogEntry"]
# [/DEF:TaskManagerPackage:Module]
# [/DEF:TaskManagerPackage:Module]

4
backend/src/core/task_manager/context.py
View File

@@ -3,7 +3,7 @@
# @PURPOSE: Provides execution context passed to plugins during task execution.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> [TaskLoggerModule]
# @RELATION: USED_BY -> [TaskManager]
# @RELATION: DEPENDS_ON -> [TaskManager]
# @COMPLEXITY: 5
# @INVARIANT: Each TaskContext is bound to a single task execution.
# @PRE: Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries.
@@ -31,7 +31,7 @@ from ..logger import belief_scope
# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# @UX_STATE: Idle -> Active -> Complete
#
# @TEST_CONTRACT: TaskContextInit ->
# @TEST_CONTRACT: TaskContextContract ->
# {
# required_fields: {task_id: str, add_log_fn: Callable, params: dict},
# optional_fields: {default_source: str},

109
backend/src/core/task_manager/manager.py
View File

@@ -1,4 +1,4 @@
# [DEF:TaskManager:Module]
# [DEF:TaskManagerModule:Module]
# @COMPLEXITY: 5
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
@@ -10,9 +10,13 @@
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @INVARIANT: Task IDs are unique.
# @CONSTRAINT: Must use belief_scope for logging.
# @TEST_CONTRACT: TaskManagerModule -> {
# @TEST_CONTRACT: TaskManagerRuntime -> {
# required_fields: {plugin_loader: PluginLoader},
# optional_fields: {},
# invariants: ["Must use belief_scope for logging"]
@@ -47,6 +51,12 @@ from ..logger import logger, belief_scope, should_log_task_level
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration.
# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.
# @INVARIANT: Task IDs are unique within the registry.
# @INVARIANT: Each task has exactly one status at any time.
# @INVARIANT: Log entries are never deleted after being added to a task.
@@ -60,6 +70,44 @@ class TaskManager:
# Log flush interval in seconds
LOG_FLUSH_INTERVAL = 2.0
# [DEF:TaskGraph:Block]
# @COMPLEXITY: 5
# @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.
# @RELATION: [DEPENDS_ON] ->[Task]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.
# @POST: Each live task id resolves to at most one active Task node and optional pause future.
# @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup.
# @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]
# @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels.
# [/DEF:TaskGraph:Block]
# [DEF:EventBus:Block]
# @COMPLEXITY: 5
# @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.
# @RELATION: [DEPENDS_ON] ->[LogEntry]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.
# @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.
# @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.
# @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]
# @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.
# [/DEF:EventBus:Block]
# [DEF:JobLifecycle:Block]
# @COMPLEXITY: 5
# @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.
# @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.
# @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.
# @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]
# @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.
# [/DEF:JobLifecycle:Block]
# [DEF:__init__:Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize the TaskManager with dependencies.
@@ -67,9 +115,12 @@ class TaskManager:
# @POST: TaskManager is ready to accept tasks.
# @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @PARAM: plugin_loader - The plugin loader instance.
def __init__(self, plugin_loader):
with belief_scope("TaskManager.__init__"):
logger.reason("Initializing task manager runtime services")
self.plugin_loader = plugin_loader
self.tasks: Dict[str, Task] = {}
self.subscribers: Dict[str, List[asyncio.Queue]] = {}
@@ -98,6 +149,10 @@ class TaskManager:
# Load persisted tasks on startup
self.load_persisted_tasks()
logger.reflect(
"Task manager runtime initialized",
extra={"task_count": len(self.tasks)},
)
# [/DEF:__init__:Function]
@@ -107,6 +162,7 @@ class TaskManager:
# @PRE: TaskManager is initialized.
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
# @RELATION: [CALLS] ->[TaskManager._flush_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flusher_loop(self):
"""Background thread that flushes log buffer to database."""
while not self._flusher_stop_event.is_set():
@@ -121,6 +177,7 @@ class TaskManager:
# @PRE: None.
# @POST: All buffered logs are written to task_logs table.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flush_logs(self):
"""Flush all buffered logs to the database."""
with self._log_buffer_lock:
@@ -151,6 +208,7 @@ class TaskManager:
# @POST: Task's buffered logs are written to database.
# @PARAM: task_id (str) - The task ID.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flush_task_logs(self, task_id: str):
"""Flush logs for a specific task immediately."""
with belief_scope("_flush_task_logs"):
@@ -176,6 +234,8 @@ class TaskManager:
# @RETURN: Task - The created task instance.
# @THROWS: ValueError if plugin not found or params invalid.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
async def create_task(
self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None
) -> Task:
@@ -190,6 +250,7 @@ class TaskManager:
logger.error("Task parameters must be a dictionary.")
raise ValueError("Task parameters must be a dictionary.")
logger.reason("Creating task node and scheduling execution")
task = Task(plugin_id=plugin_id, params=params, user_id=user_id)
self.tasks[task.id] = task
self.persistence_service.persist_task(task)
@@ -197,6 +258,10 @@ class TaskManager:
self.loop.create_task(
self._run_task(task.id)
) # Schedule task for execution
logger.reflect(
"Task creation persisted and execution scheduled",
extra={"task_id": task.id, "plugin_id": plugin_id},
)
return task
# [/DEF:create_task:Function]
@@ -208,11 +273,18 @@ class TaskManager:
# @POST: Task is executed, status updated to SUCCESS or FAILED.
# @PARAM: task_id (str) - The ID of the task to run.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[EventBus]
async def _run_task(self, task_id: str):
with belief_scope("TaskManager._run_task", f"task_id={task_id}"):
task = self.tasks[task_id]
plugin = self.plugin_loader.get_plugin(task.plugin_id)
logger.reason(
"Transitioning task to running state",
extra={"task_id": task_id, "plugin_id": task.plugin_id},
)
logger.info(
f"Starting execution of task {task_id} for plugin '{plugin.name}'"
)
@@ -286,6 +358,10 @@ class TaskManager:
logger.info(
f"Task {task_id} execution finished with status: {task.status}"
)
logger.reflect(
"Task lifecycle reached persisted terminal state",
extra={"task_id": task_id, "status": str(task.status)},
)
# [/DEF:_run_task:Function]
@@ -298,6 +374,7 @@ class TaskManager:
# @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.
# @THROWS: ValueError if task not found or not awaiting mapping.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):
with belief_scope("TaskManager.resolve_task", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -323,6 +400,7 @@ class TaskManager:
# @POST: Execution pauses until future is set.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
async def wait_for_resolution(self, task_id: str):
with belief_scope("TaskManager.wait_for_resolution", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -347,7 +425,7 @@ class TaskManager:
# @PRE: Task exists.
# @POST: Execution pauses until future is set via resume_task_with_password.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[asyncio.AbstractEventLoop.create_future]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
async def wait_for_input(self, task_id: str):
with belief_scope("TaskManager.wait_for_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -372,7 +450,7 @@ class TaskManager:
# @POST: Returns Task object or None.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: Optional[Task] - The task or None.
# @RELATION: [READS] ->[TaskManager.tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def get_task(self, task_id: str) -> Optional[Task]:
with belief_scope("TaskManager.get_task", f"task_id={task_id}"):
return self.tasks.get(task_id)
@@ -385,7 +463,7 @@ class TaskManager:
# @PRE: None.
# @POST: Returns list of all Task objects.
# @RETURN: List[Task] - All tasks.
# @RELATION: [READS] ->[TaskManager.tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def get_all_tasks(self) -> List[Task]:
with belief_scope("TaskManager.get_all_tasks"):
return list(self.tasks.values())
@@ -401,7 +479,7 @@ class TaskManager:
# @PARAM: offset (int) - Number of tasks to skip.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: List[Task] - List of tasks matching criteria.
# @RELATION: [READS] ->[TaskManager.tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def get_tasks(
self,
limit: int = 10,
@@ -447,6 +525,7 @@ class TaskManager:
# @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.
# @RETURN: List[LogEntry] - List of log entries.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def get_task_logs(
self, task_id: str, log_filter: Optional[LogFilter] = None
) -> List[LogEntry]:
@@ -483,6 +562,7 @@ class TaskManager:
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def get_task_log_stats(self, task_id: str) -> LogStats:
with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"):
return self.log_persistence_service.get_log_stats(task_id)
@@ -497,6 +577,7 @@ class TaskManager:
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def get_task_log_sources(self, task_id: str) -> List[str]:
with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"):
return self.log_persistence_service.get_sources(task_id)
@@ -515,6 +596,7 @@ class TaskManager:
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
# @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).
# @RELATION: [CALLS] ->[should_log_task_level]
# @RELATION: [DISPATCHES] ->[EventBus]
def _add_log(
self,
task_id: str,
@@ -565,7 +647,7 @@ class TaskManager:
# @POST: Returns an asyncio.Queue for log entries.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: asyncio.Queue - Queue for log entries.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
# @RELATION: [DEPENDS_ON] ->[EventBus]
async def subscribe_logs(self, task_id: str) -> asyncio.Queue:
with belief_scope("TaskManager.subscribe_logs", f"task_id={task_id}"):
queue = asyncio.Queue()
@@ -583,7 +665,7 @@ class TaskManager:
# @POST: Queue removed from subscribers.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: queue (asyncio.Queue) - Queue to remove.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):
with belief_scope("TaskManager.unsubscribe_logs", f"task_id={task_id}"):
if task_id in self.subscribers:
@@ -600,6 +682,7 @@ class TaskManager:
# @PRE: None.
# @POST: Persisted tasks loaded into self.tasks.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def load_persisted_tasks(self) -> None:
with belief_scope("TaskManager.load_persisted_tasks"):
loaded_tasks = self.persistence_service.load_tasks(limit=100)
@@ -618,6 +701,7 @@ class TaskManager:
# @PARAM: input_request (Dict) - Details about required input.
# @THROWS: ValueError if task not found or not RUNNING.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:
with belief_scope("TaskManager.await_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -636,7 +720,7 @@ class TaskManager:
task_id,
"INFO",
"Task paused for user input",
{"input_request": input_request},
metadata={"input_request": input_request},
)
# [/DEF:await_input:Function]
@@ -650,6 +734,7 @@ class TaskManager:
# @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.
# @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
def resume_task_with_password(
self, task_id: str, passwords: Dict[str, str]
) -> None:
@@ -676,7 +761,7 @@ class TaskManager:
task_id,
"INFO",
"Task resumed with passwords",
{"databases": list(passwords.keys())},
metadata={"databases": list(passwords.keys())},
)
if task_id in self.task_futures:
@@ -692,6 +777,8 @@ class TaskManager:
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: int - Number of tasks cleared.
# @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:
with belief_scope("TaskManager.clear_tasks"):
tasks_to_remove = []
@@ -739,4 +826,4 @@ class TaskManager:
# [/DEF:TaskManager:Class]
# [/DEF:TaskManager:Module]
# [/DEF:TaskManagerModule:Module]

16
backend/src/core/task_manager/models.py
View File

@@ -4,7 +4,7 @@
# @PURPOSE: Defines the data models and enumerations used by the Task Manager.
# @LAYER: Core
# @RELATION: [USED_BY] -> [TaskManager]
# @RELATION: [USED_BY] -> [ApiRoutes]
# @RELATION: [USED_BY] -> [TaskManagerPackage]
# @INVARIANT: Task IDs are immutable once created.
# @CONSTRAINT: Must use Pydantic for data validation.
@@ -38,7 +38,7 @@ class TaskStatus(str, Enum):
# @SEMANTICS: log, level, severity, enum
# @PURPOSE: Defines the possible log levels for task logging.
# @COMPLEXITY: 1
# @RELATION: [USED_BY] -> [LogEntry]
# @RELATION: [DEPENDS_ON] -> [LogEntry]
# @RELATION: [USED_BY] -> [TaskLogger]
class LogLevel(str, Enum):
DEBUG = "DEBUG"
@@ -55,7 +55,7 @@ class LogLevel(str, Enum):
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
# @COMPLEXITY: 2
# @RELATION: [DEPENDS_ON] -> [LogLevel]
# @RELATION: [PART_OF] -> [Task]
# @RELATION: [DEPENDS_ON] -> [Task]
# @INVARIANT: Each log entry has a unique timestamp and source.
#
# @TEST_CONTRACT: LogEntryModel ->
@@ -87,7 +87,7 @@ class LogEntry(BaseModel):
# @SEMANTICS: task, log, persistent, pydantic
# @PURPOSE: A Pydantic model representing a persisted log entry from the database.
# @COMPLEXITY: 3
# @RELATION: [MAPS_TO] -> [TaskLogRecord]
# @RELATION: [DEPENDS_ON] -> [TaskLogRecord]
class TaskLog(BaseModel):
id: int
task_id: str
@@ -108,7 +108,7 @@ class TaskLog(BaseModel):
# @SEMANTICS: log, filter, query, pydantic
# @PURPOSE: Filter parameters for querying task logs.
# @COMPLEXITY: 1
# @RELATION: [USED_BY] -> [TaskManager]
# @RELATION: [DEPENDS_ON] -> [TaskManager]
class LogFilter(BaseModel):
level: Optional[str] = None # Filter by log level
source: Optional[str] = None # Filter by source component
@@ -124,7 +124,7 @@ class LogFilter(BaseModel):
# @SEMANTICS: log, stats, aggregation, pydantic
# @PURPOSE: Statistics about log entries for a task.
# @COMPLEXITY: 1
# @RELATION: [COMPUTED_FROM] -> [TaskLog]
# @RELATION: [DEPENDS_ON] -> [TaskLog]
class LogStats(BaseModel):
total_count: int
by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2}
@@ -139,8 +139,8 @@ class LogStats(BaseModel):
# @SEMANTICS: task, job, execution, state, pydantic
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
# @RELATION: [DEPENDS_ON] -> [TaskStatus]
# @RELATION: [CONTAINS] -> [LogEntry]
# @RELATION: [USED_BY] -> [TaskManager]
# @RELATION: [DEPENDS_ON] -> [LogEntry]
# @RELATION: [DEPENDS_ON] -> [TaskManager]
class Task(BaseModel):
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
plugin_id: str

232
backend/src/core/task_manager/persistence.py
View File

@@ -7,7 +7,8 @@
# @POST: Provides reliable storage and retrieval for task metadata and logs.
# @SIDE_EFFECT: Performs database I/O on tasks.db.
# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @RELATION: [USED_BY] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @INVARIANT: Database schema must match the TaskRecord model structure.
@@ -25,6 +26,7 @@ from .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats
from ..logger import logger, belief_scope
# [/SECTION]
# [DEF:TaskPersistenceService:Class]
# @COMPLEXITY: 5
# @SEMANTICS: persistence, service, database, sqlalchemy
@@ -36,10 +38,11 @@ from ..logger import logger, belief_scope
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [DEPENDS_ON] ->[TaskRecord]
# @RELATION: [DEPENDS_ON] ->[Environment]
# @RELATION: [USED_BY] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @INVARIANT: Persistence must handle potentially missing task fields natively.
#
# @TEST_CONTRACT: TaskPersistenceService ->
# @TEST_CONTRACT: TaskPersistenceContract ->
# {
# required_fields: {},
# invariants: [
@@ -74,6 +77,7 @@ class TaskPersistenceService:
except json.JSONDecodeError:
return value
return value
# [/DEF:_json_load_if_needed:Function]
# [DEF:_parse_datetime:Function]
@@ -92,6 +96,7 @@ class TaskPersistenceService:
except ValueError:
return None
return None
# [/DEF:_parse_datetime:Function]
# [DEF:_resolve_environment_id:Function]
@@ -102,19 +107,25 @@ class TaskPersistenceService:
# @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]
# @RELATION: [DEPENDS_ON] ->[Environment]
@staticmethod
def _resolve_environment_id(session: Session, env_id: Optional[str]) -> Optional[str]:
def _resolve_environment_id(
session: Session, env_id: Optional[str]
) -> Optional[str]:
with belief_scope("_resolve_environment_id"):
raw_value = str(env_id or "").strip()
if not raw_value:
return None
# 1) Direct match by primary key.
by_id = session.query(Environment).filter(Environment.id == raw_value).first()
by_id = (
session.query(Environment).filter(Environment.id == raw_value).first()
)
if by_id:
return str(by_id.id)
# 2) Exact match by name.
by_name = session.query(Environment).filter(Environment.name == raw_value).first()
by_name = (
session.query(Environment).filter(Environment.name == raw_value).first()
)
if by_name:
return str(by_name.id)
@@ -128,10 +139,14 @@ class TaskPersistenceService:
return None
for env in session.query(Environment).all():
if normalize_token(env.id) == target_token or normalize_token(env.name) == target_token:
if (
normalize_token(env.id) == target_token
or normalize_token(env.name) == target_token
):
return str(env.id)
return None
# [/DEF:_resolve_environment_id:Function]
# [DEF:__init__:Function]
@@ -143,6 +158,7 @@ class TaskPersistenceService:
with belief_scope("TaskPersistenceService.__init__"):
# We use TasksSessionLocal from database.py
pass
# [/DEF:__init__:Function]
# [DEF:persist_task:Function]
@@ -158,18 +174,24 @@ class TaskPersistenceService:
with belief_scope("TaskPersistenceService.persist_task", f"task_id={task.id}"):
session: Session = TasksSessionLocal()
try:
record = session.query(TaskRecord).filter(TaskRecord.id == task.id).first()
record = (
session.query(TaskRecord).filter(TaskRecord.id == task.id).first()
)
if not record:
record = TaskRecord(id=task.id)
session.add(record)
record.type = task.plugin_id
record.status = task.status.value
raw_env_id = task.params.get("environment_id") or task.params.get("source_env_id")
record.environment_id = self._resolve_environment_id(session, raw_env_id)
raw_env_id = task.params.get("environment_id") or task.params.get(
"source_env_id"
)
record.environment_id = self._resolve_environment_id(
session, raw_env_id
)
record.started_at = task.started_at
record.finished_at = task.finished_at
# Ensure params and result are JSON serializable
def json_serializable(obj):
with belief_scope("TaskPersistenceService.json_serializable"):
@@ -183,31 +205,32 @@ class TaskPersistenceService:
record.params = json_serializable(task.params)
record.result = json_serializable(task.result)
# Store logs as JSON, converting datetime to string
record.logs = []
for log in task.logs:
log_dict = log.dict()
if isinstance(log_dict.get('timestamp'), datetime):
log_dict['timestamp'] = log_dict['timestamp'].isoformat()
if isinstance(log_dict.get("timestamp"), datetime):
log_dict["timestamp"] = log_dict["timestamp"].isoformat()
# Also clean up any datetimes in context
if log_dict.get('context'):
log_dict['context'] = json_serializable(log_dict['context'])
if log_dict.get("context"):
log_dict["context"] = json_serializable(log_dict["context"])
record.logs.append(log_dict)
# Extract error if failed
if task.status == TaskStatus.FAILED:
for log in reversed(task.logs):
if log.level == "ERROR":
record.error = log.message
break
session.commit()
except Exception as e:
session.rollback()
logger.error(f"Failed to persist task {task.id}: {e}")
finally:
session.close()
# [/DEF:persist_task:Function]
# [DEF:persist_tasks:Function]
@@ -221,6 +244,7 @@ class TaskPersistenceService:
with belief_scope("TaskPersistenceService.persist_tasks"):
for task in tasks:
self.persist_task(task)
# [/DEF:persist_tasks:Function]
# [DEF:load_tasks:Function]
@@ -234,16 +258,20 @@ class TaskPersistenceService:
# @DATA_CONTRACT: Model[TaskRecord] -> Output[List[Task]]
# @RELATION: [CALLS] ->[_json_load_if_needed]
# @RELATION: [CALLS] ->[_parse_datetime]
def load_tasks(self, limit: int = 100, status: Optional[TaskStatus] = None) -> List[Task]:
def load_tasks(
self, limit: int = 100, status: Optional[TaskStatus] = None
) -> List[Task]:
with belief_scope("TaskPersistenceService.load_tasks"):
session: Session = TasksSessionLocal()
try:
query = session.query(TaskRecord)
if status:
query = query.filter(TaskRecord.status == status.value)
records = query.order_by(TaskRecord.created_at.desc()).limit(limit).all()
records = (
query.order_by(TaskRecord.created_at.desc()).limit(limit).all()
)
loaded_tasks = []
for record in records:
try:
@@ -254,7 +282,10 @@ class TaskPersistenceService:
if not isinstance(log_data, dict):
continue
log_data = dict(log_data)
log_data['timestamp'] = self._parse_datetime(log_data.get('timestamp')) or datetime.utcnow()
log_data["timestamp"] = (
self._parse_datetime(log_data.get("timestamp"))
or datetime.utcnow()
)
logs.append(LogEntry(**log_data))
started_at = self._parse_datetime(record.started_at)
@@ -270,15 +301,16 @@ class TaskPersistenceService:
finished_at=finished_at,
params=params if isinstance(params, dict) else {},
result=result,
logs=logs
logs=logs,
)
loaded_tasks.append(task)
except Exception as e:
logger.error(f"Failed to reconstruct task {record.id}: {e}")
return loaded_tasks
finally:
session.close()
# [/DEF:load_tasks:Function]
# [DEF:delete_tasks:Function]
@@ -295,16 +327,22 @@ class TaskPersistenceService:
with belief_scope("TaskPersistenceService.delete_tasks"):
session: Session = TasksSessionLocal()
try:
session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(synchronize_session=False)
session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(
synchronize_session=False
)
session.commit()
except Exception as e:
session.rollback()
logger.error(f"Failed to delete tasks: {e}")
finally:
session.close()
# [/DEF:delete_tasks:Function]
# [/DEF:TaskPersistenceService:Class]
# [DEF:TaskLogPersistenceService:Class]
# @COMPLEXITY: 5
# @SEMANTICS: persistence, service, database, log, sqlalchemy
@@ -315,10 +353,11 @@ class TaskPersistenceService:
# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [USED_BY] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @INVARIANT: Log entries are batch-inserted for performance.
#
# @TEST_CONTRACT: TaskLogPersistenceService ->
# @TEST_CONTRACT: TaskLogPersistenceContract ->
# {
# required_fields: {},
# invariants: [
@@ -335,7 +374,7 @@ class TaskLogPersistenceService:
Service for persisting and querying task logs.
Supports batch inserts, filtering, and statistics.
"""
# [DEF:__init__:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initializes the TaskLogPersistenceService
@@ -343,8 +382,9 @@ class TaskLogPersistenceService:
# @POST: Service is ready for log persistence
def __init__(self, config=None):
pass
# [/DEF:__init__:Function]
# [DEF:add_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Batch insert log entries for a task.
@@ -368,7 +408,9 @@ class TaskLogPersistenceService:
level=log.level,
source=log.source or "system",
message=log.message,
metadata_json=json.dumps(log.metadata) if log.metadata else None
metadata_json=json.dumps(log.metadata)
if log.metadata
else None,
)
session.add(record)
session.commit()
@@ -377,8 +419,9 @@ class TaskLogPersistenceService:
logger.error(f"Failed to add logs for task {task_id}: {e}")
finally:
session.close()
# [/DEF:add_logs:Function]
# [DEF:get_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Query logs for a task with filtering and pagination.
@@ -395,23 +438,27 @@ class TaskLogPersistenceService:
with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
try:
query = session.query(TaskLogRecord).filter(TaskLogRecord.task_id == task_id)
query = session.query(TaskLogRecord).filter(
TaskLogRecord.task_id == task_id
)
# Apply filters
if log_filter.level:
query = query.filter(TaskLogRecord.level == log_filter.level.upper())
query = query.filter(
TaskLogRecord.level == log_filter.level.upper()
)
if log_filter.source:
query = query.filter(TaskLogRecord.source == log_filter.source)
if log_filter.search:
search_pattern = f"%{log_filter.search}%"
query = query.filter(TaskLogRecord.message.ilike(search_pattern))
# Order by timestamp ascending (oldest first)
query = query.order_by(TaskLogRecord.timestamp.asc())
# Apply pagination
records = query.offset(log_filter.offset).limit(log_filter.limit).all()
logs = []
for record in records:
metadata = None
@@ -420,22 +467,25 @@ class TaskLogPersistenceService:
metadata = json.loads(record.metadata_json)
except json.JSONDecodeError:
metadata = None
logs.append(TaskLog(
id=record.id,
task_id=record.task_id,
timestamp=record.timestamp,
level=record.level,
source=record.source,
message=record.message,
metadata=metadata
))
logs.append(
TaskLog(
id=record.id,
task_id=record.task_id,
timestamp=record.timestamp,
level=record.level,
source=record.source,
message=record.message,
metadata=metadata,
)
)
return logs
finally:
session.close()
# [/DEF:get_logs:Function]
# [DEF:get_log_stats:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get statistics about logs for a task.
@@ -447,44 +497,48 @@ class TaskLogPersistenceService:
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[LogStats]
def get_log_stats(self, task_id: str) -> LogStats:
with belief_scope("TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"):
with belief_scope(
"TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"
):
session: Session = TasksSessionLocal()
try:
# Get total count
total_count = session.query(TaskLogRecord).filter(
TaskLogRecord.task_id == task_id
).count()
total_count = (
session.query(TaskLogRecord)
.filter(TaskLogRecord.task_id == task_id)
.count()
)
# Get counts by level
from sqlalchemy import func
level_counts = session.query(
TaskLogRecord.level,
func.count(TaskLogRecord.id)
).filter(
TaskLogRecord.task_id == task_id
).group_by(TaskLogRecord.level).all()
level_counts = (
session.query(TaskLogRecord.level, func.count(TaskLogRecord.id))
.filter(TaskLogRecord.task_id == task_id)
.group_by(TaskLogRecord.level)
.all()
)
by_level = {level: count for level, count in level_counts}
# Get counts by source
source_counts = session.query(
TaskLogRecord.source,
func.count(TaskLogRecord.id)
).filter(
TaskLogRecord.task_id == task_id
).group_by(TaskLogRecord.source).all()
source_counts = (
session.query(TaskLogRecord.source, func.count(TaskLogRecord.id))
.filter(TaskLogRecord.task_id == task_id)
.group_by(TaskLogRecord.source)
.all()
)
by_source = {source: count for source, count in source_counts}
return LogStats(
total_count=total_count,
by_level=by_level,
by_source=by_source
total_count=total_count, by_level=by_level, by_source=by_source
)
finally:
session.close()
# [/DEF:get_log_stats:Function]
# [DEF:get_sources:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get unique sources for a task's logs.
@@ -495,18 +549,24 @@ class TaskLogPersistenceService:
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def get_sources(self, task_id: str) -> List[str]:
with belief_scope("TaskLogPersistenceService.get_sources", f"task_id={task_id}"):
with belief_scope(
"TaskLogPersistenceService.get_sources", f"task_id={task_id}"
):
session: Session = TasksSessionLocal()
try:
from sqlalchemy import distinct
sources = session.query(distinct(TaskLogRecord.source)).filter(
TaskLogRecord.task_id == task_id
).all()
sources = (
session.query(distinct(TaskLogRecord.source))
.filter(TaskLogRecord.task_id == task_id)
.all()
)
return [s[0] for s in sources]
finally:
session.close()
# [/DEF:get_sources:Function]
# [DEF:delete_logs_for_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Delete all logs for a specific task.
@@ -516,7 +576,9 @@ class TaskLogPersistenceService:
# @SIDE_EFFECT: Deletes from task_logs table.
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def delete_logs_for_task(self, task_id: str) -> None:
with belief_scope("TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"):
with belief_scope(
"TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"
):
session: Session = TasksSessionLocal()
try:
session.query(TaskLogRecord).filter(
@@ -528,8 +590,9 @@ class TaskLogPersistenceService:
logger.error(f"Failed to delete logs for task {task_id}: {e}")
finally:
session.close()
# [/DEF:delete_logs_for_task:Function]
# [DEF:delete_logs_for_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Delete all logs for multiple tasks.
@@ -553,6 +616,9 @@ class TaskLogPersistenceService:
logger.error(f"Failed to delete logs for tasks: {e}")
finally:
session.close()
# [/DEF:delete_logs_for_tasks:Function]
# [/DEF:TaskLogPersistenceService:Class]
# [/DEF:TaskPersistenceModule:Module]

8
backend/src/core/task_manager/task_logger.py
View File

@@ -3,7 +3,7 @@
# @PURPOSE: Provides a dedicated logger for tasks with automatic source attribution.
# @LAYER: Core
# @RELATION: [DEPENDS_ON] -> [TaskManager]
# @RELATION: [CALLS] -> [_add_log]
# @RELATION: [DEPENDS_ON] -> [EventBus]
# @COMPLEXITY: 2
# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source.
@@ -17,12 +17,12 @@ from typing import Dict, Any, Optional, Callable
# @PURPOSE: A wrapper around TaskManager._add_log that carries task_id and source context.
# @COMPLEXITY: 2
# @RELATION: [DEPENDS_ON] -> [TaskManager]
# @RELATION: [CALLS] -> [_add_log]
# @RELATION: [USED_BY] -> [TaskManager]
# @RELATION: [DEPENDS_ON] -> [EventBus]
# @RELATION: [USED_BY] -> [TaskContext]
# @INVARIANT: All log calls include the task_id and source.
# @UX_STATE: Idle -> Logging -> (system records log)
#
# @TEST_CONTRACT: TaskLoggerModel ->
# @TEST_CONTRACT: TaskLoggerContract ->
# {
# required_fields: {task_id: str, add_log_fn: Callable},
# optional_fields: {source: str},

157
backend/src/models/__tests__/test_clean_release.py
View File

@@ -1,16 +1,16 @@
# [DEF:__tests__/test_clean_release:Module]
# @RELATION: VERIFIES -> ../clean_release.py
# [DEF:TestCleanReleaseModels:Module]
# @RELATION: VERIFIES -> [CleanReleaseModels]
# @PURPOSE: Contract testing for Clean Release models
# [/DEF:__tests__/test_clean_release:Module]
# [/DEF:TestCleanReleaseModels:Module]
import pytest
from datetime import datetime
from pydantic import ValidationError
from src.models.clean_release import (
ReleaseCandidate,
ReleaseCandidateStatus,
ProfileType,
CleanProfilePolicy,
ReleaseCandidate,
ReleaseCandidateStatus,
ProfileType,
CleanProfilePolicy,
DistributionManifest,
ManifestItem,
ManifestSummary,
@@ -21,9 +21,10 @@ from src.models.clean_release import (
CheckStageName,
CheckStageStatus,
ComplianceReport,
ExecutionMode
ExecutionMode,
)
# @TEST_FIXTURE: valid_enterprise_candidate
@pytest.fixture
def valid_candidate_data():
@@ -33,30 +34,35 @@ def valid_candidate_data():
"profile": ProfileType.ENTERPRISE_CLEAN,
"created_at": datetime.now(),
"created_by": "admin",
"source_snapshot_ref": "v1.0.0-snapshot"
"source_snapshot_ref": "v1.0.0-snapshot",
}
# [DEF:test_release_candidate_valid:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify that a valid release candidate can be instantiated.
def test_release_candidate_valid(valid_candidate_data):
rc = ReleaseCandidate(**valid_candidate_data)
assert rc.candidate_id == "RC-001"
assert rc.status == ReleaseCandidateStatus.DRAFT
# [/DEF:test_release_candidate_valid:Function]
# [DEF:test_release_candidate_empty_id:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify that a release candidate with an empty ID is rejected.
def test_release_candidate_empty_id(valid_candidate_data):
valid_candidate_data["candidate_id"] = " "
with pytest.raises(ValueError, match="candidate_id must be non-empty"):
ReleaseCandidate(**valid_candidate_data)
# @TEST_FIXTURE: valid_enterprise_policy
# [/DEF:test_release_candidate_empty_id:Function]
@pytest.fixture
def valid_policy_data():
return {
@@ -67,72 +73,103 @@ def valid_policy_data():
"required_system_categories": ["core"],
"internal_source_registry_ref": "REG-1",
"effective_from": datetime.now(),
"profile": ProfileType.ENTERPRISE_CLEAN
"profile": ProfileType.ENTERPRISE_CLEAN,
}
# @TEST_INVARIANT: policy_purity
# [DEF:test_enterprise_policy_valid:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify that a valid enterprise policy is accepted.
def test_enterprise_policy_valid(valid_policy_data):
policy = CleanProfilePolicy(**valid_policy_data)
assert policy.external_source_forbidden is True
# @TEST_EDGE: enterprise_policy_missing_prohibited
# [/DEF:test_enterprise_policy_valid:Function]
# [DEF:test_enterprise_policy_missing_prohibited:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify that an enterprise policy without prohibited categories is rejected.
def test_enterprise_policy_missing_prohibited(valid_policy_data):
valid_policy_data["prohibited_artifact_categories"] = []
with pytest.raises(ValueError, match="enterprise-clean policy requires prohibited_artifact_categories"):
with pytest.raises(
ValueError,
match="enterprise-clean policy requires prohibited_artifact_categories",
):
CleanProfilePolicy(**valid_policy_data)
# @TEST_EDGE: enterprise_policy_external_allowed
# [/DEF:test_enterprise_policy_missing_prohibited:Function]
# [DEF:test_enterprise_policy_external_allowed:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify that an enterprise policy allowing external sources is rejected.
def test_enterprise_policy_external_allowed(valid_policy_data):
valid_policy_data["external_source_forbidden"] = False
with pytest.raises(ValueError, match="enterprise-clean policy requires external_source_forbidden=true"):
with pytest.raises(
ValueError,
match="enterprise-clean policy requires external_source_forbidden=true",
):
CleanProfilePolicy(**valid_policy_data)
# @TEST_INVARIANT: manifest_consistency
# @TEST_EDGE: manifest_count_mismatch
# [/DEF:test_enterprise_policy_external_allowed:Function]
# [DEF:test_manifest_count_mismatch:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify that a manifest with count mismatches is rejected.
def test_manifest_count_mismatch():
summary = ManifestSummary(included_count=1, excluded_count=0, prohibited_detected_count=0)
item = ManifestItem(path="p", category="c", classification=ClassificationType.ALLOWED, reason="r")
summary = ManifestSummary(
included_count=1, excluded_count=0, prohibited_detected_count=0
)
item = ManifestItem(
path="p", category="c", classification=ClassificationType.ALLOWED, reason="r"
)
# Valid
DistributionManifest(
manifest_id="m1", candidate_id="rc1", policy_id="p1",
generated_at=datetime.now(), generated_by="u", items=[item],
summary=summary, deterministic_hash="h"
manifest_id="m1",
candidate_id="rc1",
policy_id="p1",
generated_at=datetime.now(),
generated_by="u",
items=[item],
summary=summary,
deterministic_hash="h",
)
# Invalid count
summary.included_count = 2
with pytest.raises(ValueError, match="manifest summary counts must match items size"):
with pytest.raises(
ValueError, match="manifest summary counts must match items size"
):
DistributionManifest(
manifest_id="m1", candidate_id="rc1", policy_id="p1",
generated_at=datetime.now(), generated_by="u", items=[item],
summary=summary, deterministic_hash="h"
manifest_id="m1",
candidate_id="rc1",
policy_id="p1",
generated_at=datetime.now(),
generated_by="u",
items=[item],
summary=summary,
deterministic_hash="h",
)
# @TEST_INVARIANT: run_integrity
# @TEST_EDGE: compliant_run_stage_fail
# [/DEF:test_manifest_count_mismatch:Function]
# [DEF:test_compliant_run_validation:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify compliant run validation logic and mandatory stage checks.
def test_compliant_run_validation():
base_run = {
@@ -144,45 +181,69 @@ def test_compliant_run_validation():
"execution_mode": ExecutionMode.TUI,
"final_status": CheckFinalStatus.COMPLIANT,
"checks": [
CheckStageResult(stage=CheckStageName.DATA_PURITY, status=CheckStageStatus.PASS),
CheckStageResult(stage=CheckStageName.INTERNAL_SOURCES_ONLY, status=CheckStageStatus.PASS),
CheckStageResult(stage=CheckStageName.NO_EXTERNAL_ENDPOINTS, status=CheckStageStatus.PASS),
CheckStageResult(stage=CheckStageName.MANIFEST_CONSISTENCY, status=CheckStageStatus.PASS),
]
CheckStageResult(
stage=CheckStageName.DATA_PURITY, status=CheckStageStatus.PASS
),
CheckStageResult(
stage=CheckStageName.INTERNAL_SOURCES_ONLY, status=CheckStageStatus.PASS
),
CheckStageResult(
stage=CheckStageName.NO_EXTERNAL_ENDPOINTS, status=CheckStageStatus.PASS
),
CheckStageResult(
stage=CheckStageName.MANIFEST_CONSISTENCY, status=CheckStageStatus.PASS
),
],
}
# Valid
ComplianceCheckRun(**base_run)
# One stage fails -> cannot be COMPLIANT
base_run["checks"][0].status = CheckStageStatus.FAIL
with pytest.raises(ValueError, match="compliant run requires PASS on all mandatory stages"):
with pytest.raises(
ValueError, match="compliant run requires PASS on all mandatory stages"
):
ComplianceCheckRun(**base_run)
# Missing stage -> cannot be COMPLIANT
base_run["checks"] = base_run["checks"][1:]
with pytest.raises(ValueError, match="compliant run requires all mandatory stages"):
ComplianceCheckRun(**base_run)
# [/DEF:test_compliant_run_validation:Function]
# [DEF:test_report_validation:Function]
# @RELATION: BINDS_TO -> __tests__/test_clean_release
# @RELATION: BINDS_TO -> [TestCleanReleaseModels]
# @PURPOSE: Verify compliance report validation based on status and violation counts.
def test_report_validation():
# Valid blocked report
ComplianceReport(
report_id="rep1", check_run_id="run1", candidate_id="rc1",
generated_at=datetime.now(), final_status=CheckFinalStatus.BLOCKED,
operator_summary="Blocked", structured_payload_ref="ref",
violations_count=2, blocking_violations_count=2
report_id="rep1",
check_run_id="run1",
candidate_id="rc1",
generated_at=datetime.now(),
final_status=CheckFinalStatus.BLOCKED,
operator_summary="Blocked",
structured_payload_ref="ref",
violations_count=2,
blocking_violations_count=2,
)
# BLOCKED with 0 blocking violations
with pytest.raises(ValueError, match="blocked report requires blocking violations"):
ComplianceReport(
report_id="rep1", check_run_id="run1", candidate_id="rc1",
generated_at=datetime.now(), final_status=CheckFinalStatus.BLOCKED,
operator_summary="Blocked", structured_payload_ref="ref",
violations_count=2, blocking_violations_count=0
report_id="rep1",
check_run_id="run1",
candidate_id="rc1",
generated_at=datetime.now(),
final_status=CheckFinalStatus.BLOCKED,
operator_summary="Blocked",
structured_payload_ref="ref",
violations_count=2,
blocking_violations_count=0,
)
# [/DEF:test_report_validation:Function]

6
backend/src/models/__tests__/test_models.py
View File

@@ -2,7 +2,7 @@
# @COMPLEXITY: 1
# @PURPOSE: Unit tests for data models
# @LAYER: Domain
# @RELATION: VERIFIES -> src.models
# @RELATION: VERIFIES -> [ModelsPackage]
import sys
from pathlib import Path
@@ -26,11 +26,13 @@ def test_environment_model():
name="test-env",
url="http://localhost:8088/api/v1",
username="admin",
password="password"
password="password",
)
assert env.id == "test-id"
assert env.name == "test-env"
assert env.url == "http://localhost:8088/api/v1"
# [/DEF:test_environment_model:Function]

4
backend/src/schemas/__init__.py
View File

@@ -1,3 +1,3 @@
# [DEF:src.schemas:Package]
# [DEF:SchemasPackage:Package]
# @PURPOSE: API schema package root.
# [/DEF:src.schemas:Package]
# [/DEF:SchemasPackage:Package]

42
backend/src/schemas/auth.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.schemas.auth:Module]
# [DEF:AuthSchemas:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: auth, schemas, pydantic, user, token
@@ -14,22 +14,29 @@ from pydantic import BaseModel, EmailStr
from datetime import datetime
# [/SECTION]
# [DEF:Token:Class]
# @COMPLEXITY: 1
# @PURPOSE: Represents a JWT access token response.
class Token(BaseModel):
access_token: str
token_type: str
# [/DEF:Token:Class]
# [DEF:TokenData:Class]
# @COMPLEXITY: 1
# @PURPOSE: Represents the data encoded in a JWT token.
class TokenData(BaseModel):
username: Optional[str] = None
scopes: List[str] = []
# [/DEF:TokenData:Class]
# [DEF:PermissionSchema:Class]
# @COMPLEXITY: 1
# @PURPOSE: Represents a permission in API responses.
@@ -40,8 +47,11 @@ class PermissionSchema(BaseModel):
class Config:
from_attributes = True
# [/DEF:PermissionSchema:Class]
# [DEF:RoleSchema:Class]
# @PURPOSE: Represents a role in API responses.
class RoleSchema(BaseModel):
@@ -52,24 +62,33 @@ class RoleSchema(BaseModel):
class Config:
from_attributes = True
# [/DEF:RoleSchema:Class]
# [DEF:RoleCreate:Class]
# @PURPOSE: Schema for creating a new role.
class RoleCreate(BaseModel):
name: str
description: Optional[str] = None
permissions: List[str] = [] # List of permission IDs or "resource:action" strings
permissions: List[str] = [] # List of permission IDs or "resource:action" strings
# [/DEF:RoleCreate:Class]
# [DEF:RoleUpdate:Class]
# @PURPOSE: Schema for updating an existing role.
class RoleUpdate(BaseModel):
name: Optional[str] = None
description: Optional[str] = None
permissions: Optional[List[str]] = None
# [/DEF:RoleUpdate:Class]
# [DEF:ADGroupMappingSchema:Class]
# @PURPOSE: Represents an AD Group to Role mapping in API responses.
class ADGroupMappingSchema(BaseModel):
@@ -79,30 +98,42 @@ class ADGroupMappingSchema(BaseModel):
class Config:
from_attributes = True
# [/DEF:ADGroupMappingSchema:Class]
# [DEF:ADGroupMappingCreate:Class]
# @PURPOSE: Schema for creating an AD Group mapping.
class ADGroupMappingCreate(BaseModel):
ad_group: str
role_id: str
# [/DEF:ADGroupMappingCreate:Class]
# [DEF:UserBase:Class]
# @PURPOSE: Base schema for user data.
class UserBase(BaseModel):
username: str
email: Optional[EmailStr] = None
is_active: bool = True
# [/DEF:UserBase:Class]
# [DEF:UserCreate:Class]
# @PURPOSE: Schema for creating a new user.
class UserCreate(UserBase):
password: str
roles: List[str] = []
# [/DEF:UserCreate:Class]
# [DEF:UserUpdate:Class]
# @PURPOSE: Schema for updating an existing user.
class UserUpdate(BaseModel):
@@ -110,8 +141,11 @@ class UserUpdate(BaseModel):
password: Optional[str] = None
is_active: Optional[bool] = None
roles: Optional[List[str]] = None
# [/DEF:UserUpdate:Class]
# [DEF:User:Class]
# @PURPOSE: Schema for user data in API responses.
class User(UserBase):
@@ -123,6 +157,8 @@ class User(UserBase):
class Config:
from_attributes = True
# [/DEF:User:Class]
# [/DEF:backend.src.schemas.auth:Module]
# [/DEF:AuthSchemas:Module]

10
backend/src/schemas/health.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.schemas.health:Module]
# [DEF:HealthSchemas:Module]
# @COMPLEXITY: 3
# @SEMANTICS: health, schemas, pydantic
# @PURPOSE: Pydantic schemas for dashboard health summary.
@@ -8,6 +8,7 @@ from pydantic import BaseModel, Field
from typing import List, Optional
from datetime import datetime
# [DEF:DashboardHealthItem:Class]
# @PURPOSE: Represents the latest health status of a single dashboard.
class DashboardHealthItem(BaseModel):
@@ -20,8 +21,11 @@ class DashboardHealthItem(BaseModel):
last_check: datetime
task_id: Optional[str] = None
summary: Optional[str] = None
# [/DEF:DashboardHealthItem:Class]
# [DEF:HealthSummaryResponse:Class]
# @PURPOSE: Aggregated health summary for all dashboards.
class HealthSummaryResponse(BaseModel):
@@ -30,6 +34,8 @@ class HealthSummaryResponse(BaseModel):
warn_count: int
fail_count: int
unknown_count: int
# [/DEF:HealthSummaryResponse:Class]
# [/DEF:backend.src.schemas.health:Module]
# [/DEF:HealthSchemas:Module]

30
backend/src/schemas/profile.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.schemas.profile:Module]
# [DEF:ProfileSchemas:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: profile, schemas, pydantic, preferences, superset, lookup, security, git, ux
@@ -21,6 +21,8 @@ from pydantic import BaseModel, Field
class ProfilePermissionState(BaseModel):
key: str
allowed: bool
# [/DEF:ProfilePermissionState:Class]
@@ -34,6 +36,8 @@ class ProfileSecuritySummary(BaseModel):
role_source: Optional[str] = None
roles: List[str] = Field(default_factory=list)
permissions: List[ProfilePermissionState] = Field(default_factory=list)
# [/DEF:ProfileSecuritySummary:Class]
@@ -65,6 +69,8 @@ class ProfilePreference(BaseModel):
class Config:
from_attributes = True
# [/DEF:ProfilePreference:Class]
@@ -106,11 +112,11 @@ class ProfilePreferenceUpdateRequest(BaseModel):
default=None,
description="Auto-open task drawer when long-running tasks start.",
)
dashboards_table_density: Optional[
Literal["compact", "comfortable", "free"]
] = Field(
default=None,
description="Preferred table density for dashboard listings.",
dashboards_table_density: Optional[Literal["compact", "comfortable", "free"]] = (
Field(
default=None,
description="Preferred table density for dashboard listings.",
)
)
telegram_id: Optional[str] = Field(
default=None,
@@ -124,6 +130,8 @@ class ProfilePreferenceUpdateRequest(BaseModel):
default=None,
description="Whether to send notifications on validation failure.",
)
# [/DEF:ProfilePreferenceUpdateRequest:Class]
@@ -136,6 +144,8 @@ class ProfilePreferenceResponse(BaseModel):
validation_errors: List[str] = Field(default_factory=list)
preference: ProfilePreference
security: ProfileSecuritySummary = Field(default_factory=ProfileSecuritySummary)
# [/DEF:ProfilePreferenceResponse:Class]
@@ -149,6 +159,8 @@ class SupersetAccountLookupRequest(BaseModel):
page_size: int = Field(default=20, ge=1, le=100)
sort_column: str = Field(default="username")
sort_order: str = Field(default="desc")
# [/DEF:SupersetAccountLookupRequest:Class]
@@ -161,6 +173,8 @@ class SupersetAccountCandidate(BaseModel):
display_name: Optional[str] = None
email: Optional[str] = None
is_active: Optional[bool] = None
# [/DEF:SupersetAccountCandidate:Class]
@@ -175,6 +189,8 @@ class SupersetAccountLookupResponse(BaseModel):
total: int = Field(ge=0)
warning: Optional[str] = None
items: List[SupersetAccountCandidate] = Field(default_factory=list)
# [/DEF:SupersetAccountLookupResponse:Class]
# [/DEF:backend.src.schemas.profile:Module]
# [/DEF:ProfileSchemas:Module]

44
backend/src/schemas/settings.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.schemas.settings:Module]
# [DEF:SettingsSchemas:Module]
# @COMPLEXITY: 3
# @SEMANTICS: settings, schemas, pydantic, validation
# @PURPOSE: Pydantic schemas for application settings and automation policies.
@@ -8,37 +8,60 @@ from pydantic import BaseModel, Field
from typing import List, Optional
from datetime import datetime, time
# [DEF:NotificationChannel:Class]
# @PURPOSE: Structured notification channel definition for policy-level custom routing.
class NotificationChannel(BaseModel):
type: str = Field(..., description="Notification channel type (e.g., SLACK, SMTP, TELEGRAM)")
target: str = Field(..., description="Notification destination (e.g., #alerts, chat id, email)")
type: str = Field(
..., description="Notification channel type (e.g., SLACK, SMTP, TELEGRAM)"
)
target: str = Field(
..., description="Notification destination (e.g., #alerts, chat id, email)"
)
# [/DEF:NotificationChannel:Class]
# [DEF:ValidationPolicyBase:Class]
# @PURPOSE: Base schema for validation policy data.
class ValidationPolicyBase(BaseModel):
name: str = Field(..., description="Name of the policy")
environment_id: str = Field(..., description="Target Superset environment ID")
is_active: bool = Field(True, description="Whether the policy is currently active")
dashboard_ids: List[str] = Field(..., description="List of dashboard IDs to validate")
schedule_days: List[int] = Field(..., description="Days of the week (0-6, 0=Sunday) to run")
dashboard_ids: List[str] = Field(
..., description="List of dashboard IDs to validate"
)
schedule_days: List[int] = Field(
..., description="Days of the week (0-6, 0=Sunday) to run"
)
window_start: time = Field(..., description="Start of the execution window")
window_end: time = Field(..., description="End of the execution window")
notify_owners: bool = Field(True, description="Whether to notify dashboard owners on failure")
notify_owners: bool = Field(
True, description="Whether to notify dashboard owners on failure"
)
custom_channels: Optional[List[NotificationChannel]] = Field(
None,
description="List of additional structured notification channels",
)
alert_condition: str = Field("FAIL_ONLY", description="Condition to trigger alerts: FAIL_ONLY, WARN_AND_FAIL, ALWAYS")
alert_condition: str = Field(
"FAIL_ONLY",
description="Condition to trigger alerts: FAIL_ONLY, WARN_AND_FAIL, ALWAYS",
)
# [/DEF:ValidationPolicyBase:Class]
# [DEF:ValidationPolicyCreate:Class]
# @PURPOSE: Schema for creating a new validation policy.
class ValidationPolicyCreate(ValidationPolicyBase):
pass
# [/DEF:ValidationPolicyCreate:Class]
# [DEF:ValidationPolicyUpdate:Class]
# @PURPOSE: Schema for updating an existing validation policy.
class ValidationPolicyUpdate(BaseModel):
@@ -52,8 +75,11 @@ class ValidationPolicyUpdate(BaseModel):
notify_owners: Optional[bool] = None
custom_channels: Optional[List[NotificationChannel]] = None
alert_condition: Optional[str] = None
# [/DEF:ValidationPolicyUpdate:Class]
# [DEF:ValidationPolicyResponse:Class]
# @PURPOSE: Schema for validation policy response data.
class ValidationPolicyResponse(ValidationPolicyBase):
@@ -63,6 +89,8 @@ class ValidationPolicyResponse(ValidationPolicyBase):
class Config:
from_attributes = True
# [/DEF:ValidationPolicyResponse:Class]
# [/DEF:backend.src.schemas.settings:Module]
# [/DEF:SettingsSchemas:Module]

4
backend/src/scripts/__init__.py
View File

@@ -1,3 +1,3 @@
# [DEF:src.scripts:Package]
# [DEF:ScriptsPackage:Package]
# @PURPOSE: Script entrypoint package root.
# [/DEF:src.scripts:Package]
# [/DEF:ScriptsPackage:Package]

88
backend/src/scripts/clean_release_cli.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.scripts.clean_release_cli:Module]
# [DEF:CleanReleaseCliScript:Module]
# @COMPLEXITY: 3
# @SEMANTICS: cli, clean-release, candidate, artifacts, manifest
# @PURPOSE: Provide headless CLI commands for candidate registration, artifact import and manifest build.
@@ -12,10 +12,18 @@ from datetime import date, datetime, timezone
from typing import Any, Dict, List, Optional
from ..models.clean_release import CandidateArtifact, ReleaseCandidate
from ..services.clean_release.approval_service import approve_candidate, reject_candidate
from ..services.clean_release.compliance_execution_service import ComplianceExecutionService
from ..services.clean_release.approval_service import (
approve_candidate,
reject_candidate,
)
from ..services.clean_release.compliance_execution_service import (
ComplianceExecutionService,
)
from ..services.clean_release.enums import CandidateStatus
from ..services.clean_release.publication_service import publish_candidate, revoke_publication
from ..services.clean_release.publication_service import (
publish_candidate,
revoke_publication,
)
# [DEF:build_parser:Function]
@@ -88,6 +96,8 @@ def build_parser() -> argparse.ArgumentParser:
revoke.add_argument("--json", action="store_true")
return parser
# [/DEF:build_parser:Function]
@@ -97,6 +107,7 @@ def build_parser() -> argparse.ArgumentParser:
# @POST: Candidate is persisted in DRAFT status.
def run_candidate_register(args: argparse.Namespace) -> int:
from ..dependencies import get_clean_release_repository
repository = get_clean_release_repository()
existing = repository.get_candidate(args.candidate_id)
if existing is not None:
@@ -114,6 +125,8 @@ def run_candidate_register(args: argparse.Namespace) -> int:
repository.save_candidate(candidate)
print(json.dumps({"status": "ok", "candidate_id": candidate.id}))
return 0
# [/DEF:run_candidate_register:Function]
@@ -123,6 +136,7 @@ def run_candidate_register(args: argparse.Namespace) -> int:
# @POST: Artifact is persisted for candidate.
def run_artifact_import(args: argparse.Namespace) -> int:
from ..dependencies import get_clean_release_repository
repository = get_clean_release_repository()
candidate = repository.get_candidate(args.candidate_id)
if candidate is None:
@@ -144,6 +158,8 @@ def run_artifact_import(args: argparse.Namespace) -> int:
print(json.dumps({"status": "ok", "artifact_id": artifact.id}))
return 0
# [/DEF:run_artifact_import:Function]
@@ -166,8 +182,18 @@ def run_manifest_build(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": str(exc)}))
return 1
print(json.dumps({"status": "ok", "manifest_id": manifest.id, "version": manifest.manifest_version}))
print(
json.dumps(
{
"status": "ok",
"manifest_id": manifest.id,
"version": manifest.manifest_version,
}
)
)
return 0
# [/DEF:run_manifest_build:Function]
@@ -180,7 +206,9 @@ def run_compliance_run(args: argparse.Namespace) -> int:
repository = get_clean_release_repository()
config_manager = get_config_manager()
service = ComplianceExecutionService(repository=repository, config_manager=config_manager)
service = ComplianceExecutionService(
repository=repository, config_manager=config_manager
)
try:
result = service.execute_run(
@@ -203,6 +231,8 @@ def run_compliance_run(args: argparse.Namespace) -> int:
}
print(json.dumps(payload))
return 0
# [/DEF:run_compliance_run:Function]
@@ -219,7 +249,9 @@ def run_compliance_status(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": "run not found"}))
return 2
report = next((item for item in repository.reports.values() if item.run_id == run.id), None)
report = next(
(item for item in repository.reports.values() if item.run_id == run.id), None
)
payload = {
"status": "ok",
"run_id": run.id,
@@ -231,6 +263,8 @@ def run_compliance_status(args: argparse.Namespace) -> int:
}
print(json.dumps(payload))
return 0
# [/DEF:run_compliance_status:Function]
@@ -259,6 +293,8 @@ def _to_payload(value: Any) -> Dict[str, Any]:
row = {column.name: getattr(value, column.name) for column in table.columns}
return _normalize(row)
raise TypeError(f"unsupported payload type: {type(value)!r}")
# [/DEF:_to_payload:Function]
@@ -275,13 +311,17 @@ def run_compliance_report(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": "run not found"}))
return 2
report = next((item for item in repository.reports.values() if item.run_id == run.id), None)
report = next(
(item for item in repository.reports.values() if item.run_id == run.id), None
)
if report is None:
print(json.dumps({"status": "error", "message": "report not found"}))
return 2
print(json.dumps({"status": "ok", "report": _to_payload(report)}))
return 0
# [/DEF:run_compliance_report:Function]
@@ -299,8 +339,14 @@ def run_compliance_violations(args: argparse.Namespace) -> int:
return 2
violations = repository.get_violations_by_run(args.run_id)
print(json.dumps({"status": "ok", "items": [_to_payload(item) for item in violations]}))
print(
json.dumps(
{"status": "ok", "items": [_to_payload(item) for item in violations]}
)
)
return 0
# [/DEF:run_compliance_violations:Function]
@@ -324,8 +370,14 @@ def run_approve(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": str(exc)}))
return 2
print(json.dumps({"status": "ok", "decision": decision.decision, "decision_id": decision.id}))
print(
json.dumps(
{"status": "ok", "decision": decision.decision, "decision_id": decision.id}
)
)
return 0
# [/DEF:run_approve:Function]
@@ -349,8 +401,14 @@ def run_reject(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": str(exc)}))
return 2
print(json.dumps({"status": "ok", "decision": decision.decision, "decision_id": decision.id}))
print(
json.dumps(
{"status": "ok", "decision": decision.decision, "decision_id": decision.id}
)
)
return 0
# [/DEF:run_reject:Function]
@@ -377,6 +435,8 @@ def run_publish(args: argparse.Namespace) -> int:
print(json.dumps({"status": "ok", "publication": _to_payload(publication)}))
return 0
# [/DEF:run_publish:Function]
@@ -401,6 +461,8 @@ def run_revoke(args: argparse.Namespace) -> int:
print(json.dumps({"status": "ok", "publication": _to_payload(publication)}))
return 0
# [/DEF:run_revoke:Function]
@@ -435,10 +497,12 @@ def main(argv: Optional[List[str]] = None) -> int:
print(json.dumps({"status": "error", "message": "unknown command"}))
return 2
# [/DEF:main:Function]
if __name__ == "__main__":
raise SystemExit(main())
# [/DEF:backend.src.scripts.clean_release_cli:Module]
# [/DEF:CleanReleaseCliScript:Module]

182
backend/src/scripts/clean_release_tui.py
View File

@@ -1,10 +1,10 @@
# [DEF:backend.src.scripts.clean_release_tui:Module]
# [DEF:CleanReleaseTuiScript:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, tui, ncurses, interactive-validator
# @PURPOSE: Interactive terminal interface for Enterprise Clean Release compliance validation.
# @LAYER: UI
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.compliance_orchestrator
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: DEPENDS_ON -> [compliance_orchestrator]
# @RELATION: DEPENDS_ON -> [repository]
# @INVARIANT: TUI refuses startup in non-TTY environments; headless flow is CLI/API only.
import curses
@@ -37,12 +37,15 @@ from src.models.clean_release import (
)
from src.services.clean_release.approval_service import approve_candidate
from src.services.clean_release.artifact_catalog_loader import load_bootstrap_artifacts
from src.services.clean_release.compliance_execution_service import ComplianceExecutionService
from src.services.clean_release.compliance_execution_service import (
ComplianceExecutionService,
)
from src.services.clean_release.enums import CandidateStatus
from src.services.clean_release.manifest_service import build_manifest_snapshot
from src.services.clean_release.publication_service import publish_candidate
from src.services.clean_release.repository import CleanReleaseRepository
# [DEF:TuiFacadeAdapter:Class]
# @PURPOSE: Thin TUI adapter that routes business mutations through application services.
# @PRE: repository contains candidate and trusted policy/registry snapshots for execution.
@@ -67,15 +70,25 @@ class TuiFacadeAdapter:
manifests = self.repository.get_manifests_by_candidate(candidate_id)
if not manifests:
raise ValueError("Manifest required before compliance run")
latest_manifest = sorted(manifests, key=lambda item: item.manifest_version, reverse=True)[0]
latest_manifest = sorted(
manifests, key=lambda item: item.manifest_version, reverse=True
)[0]
service = ComplianceExecutionService(
repository=self.repository,
config_manager=self._build_config_manager(),
)
return service.execute_run(candidate_id=candidate_id, requested_by=actor, manifest_id=latest_manifest.id)
return service.execute_run(
candidate_id=candidate_id,
requested_by=actor,
manifest_id=latest_manifest.id,
)
def approve_latest(self, *, candidate_id: str, actor: str):
reports = [item for item in self.repository.reports.values() if item.candidate_id == candidate_id]
reports = [
item
for item in self.repository.reports.values()
if item.candidate_id == candidate_id
]
if not reports:
raise ValueError("No compliance report available for approval")
report = sorted(reports, key=lambda item: item.generated_at, reverse=True)[0]
@@ -88,7 +101,11 @@ class TuiFacadeAdapter:
)
def publish_latest(self, *, candidate_id: str, actor: str):
reports = [item for item in self.repository.reports.values() if item.candidate_id == candidate_id]
reports = [
item
for item in self.repository.reports.values()
if item.candidate_id == candidate_id
]
if not reports:
raise ValueError("No compliance report available for publication")
report = sorted(reports, key=lambda item: item.generated_at, reverse=True)[0]
@@ -111,24 +128,55 @@ class TuiFacadeAdapter:
def get_overview(self, *, candidate_id: str) -> Dict[str, Any]:
candidate = self.repository.get_candidate(candidate_id)
manifests = self.repository.get_manifests_by_candidate(candidate_id)
latest_manifest = sorted(manifests, key=lambda item: item.manifest_version, reverse=True)[0] if manifests else None
runs = [item for item in self.repository.check_runs.values() if item.candidate_id == candidate_id]
latest_run = sorted(runs, key=lambda item: item.requested_at, reverse=True)[0] if runs else None
latest_report = next((item for item in self.repository.reports.values() if latest_run and item.run_id == latest_run.id), None)
latest_manifest = (
sorted(manifests, key=lambda item: item.manifest_version, reverse=True)[0]
if manifests
else None
)
runs = [
item
for item in self.repository.check_runs.values()
if item.candidate_id == candidate_id
]
latest_run = (
sorted(runs, key=lambda item: item.requested_at, reverse=True)[0]
if runs
else None
)
latest_report = next(
(
item
for item in self.repository.reports.values()
if latest_run and item.run_id == latest_run.id
),
None,
)
approvals = getattr(self.repository, "approval_decisions", [])
latest_approval = sorted(
[item for item in approvals if item.candidate_id == candidate_id],
key=lambda item: item.decided_at,
reverse=True,
)[0] if any(item.candidate_id == candidate_id for item in approvals) else None
latest_approval = (
sorted(
[item for item in approvals if item.candidate_id == candidate_id],
key=lambda item: item.decided_at,
reverse=True,
)[0]
if any(item.candidate_id == candidate_id for item in approvals)
else None
)
publications = getattr(self.repository, "publication_records", [])
latest_publication = sorted(
[item for item in publications if item.candidate_id == candidate_id],
key=lambda item: item.published_at,
reverse=True,
)[0] if any(item.candidate_id == candidate_id for item in publications) else None
latest_publication = (
sorted(
[item for item in publications if item.candidate_id == candidate_id],
key=lambda item: item.published_at,
reverse=True,
)[0]
if any(item.candidate_id == candidate_id for item in publications)
else None
)
policy = self.repository.get_active_policy()
registry = self.repository.get_registry(policy.internal_source_registry_ref) if policy else None
registry = (
self.repository.get_registry(policy.internal_source_registry_ref)
if policy
else None
)
return {
"candidate": candidate,
"manifest": latest_manifest,
@@ -139,6 +187,8 @@ class TuiFacadeAdapter:
"policy": policy,
"registry": registry,
}
# [/DEF:TuiFacadeAdapter:Class]
@@ -166,11 +216,11 @@ class CleanReleaseTUI:
curses.start_color()
curses.use_default_colors()
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) # Header/Footer
curses.init_pair(2, curses.COLOR_GREEN, -1) # PASS
curses.init_pair(3, curses.COLOR_RED, -1) # FAIL/BLOCKED
curses.init_pair(4, curses.COLOR_YELLOW, -1) # RUNNING
curses.init_pair(5, curses.COLOR_CYAN, -1) # Text
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) # Header/Footer
curses.init_pair(2, curses.COLOR_GREEN, -1) # PASS
curses.init_pair(3, curses.COLOR_RED, -1) # FAIL/BLOCKED
curses.init_pair(4, curses.COLOR_YELLOW, -1) # RUNNING
curses.init_pair(5, curses.COLOR_CYAN, -1) # Text
def _build_repository(self, mode: str) -> CleanReleaseRepository:
repo = CleanReleaseRepository()
@@ -317,7 +367,9 @@ class CleanReleaseTUI:
"prohibited_artifact_categories",
["test-data", "demo", "load-test"],
),
required_system_categories=payload.get("required_system_categories", ["core"]),
required_system_categories=payload.get(
"required_system_categories", ["core"]
),
effective_from=now,
)
)
@@ -354,18 +406,21 @@ class CleanReleaseTUI:
self.stdscr.addstr(4, 3, "Checks:")
check_defs = [
(CheckStageName.DATA_PURITY, "Data Purity (no test/demo payloads)"),
(CheckStageName.INTERNAL_SOURCES_ONLY, "Internal Sources Only (company servers)"),
(
CheckStageName.INTERNAL_SOURCES_ONLY,
"Internal Sources Only (company servers)",
),
(CheckStageName.NO_EXTERNAL_ENDPOINTS, "No External Internet Endpoints"),
(CheckStageName.MANIFEST_CONSISTENCY, "Release Manifest Consistency"),
]
row = 5
drawn_checks = {c["stage"]: c for c in self.checks_progress}
for stage, desc in check_defs:
status_text = " "
color = curses.color_pair(5)
if stage in drawn_checks:
c = drawn_checks[stage]
if c["status"] == "RUNNING":
@@ -377,7 +432,7 @@ class CleanReleaseTUI:
elif c["status"] == CheckStageStatus.FAIL:
status_text = "FAIL"
color = curses.color_pair(3)
self.stdscr.addstr(row, 4, f"[{status_text:^4}] {desc}")
if status_text != " ":
self.stdscr.addstr(row, 50, f"{status_text:>10}", color | curses.A_BOLD)
@@ -396,12 +451,18 @@ class CleanReleaseTUI:
def draw_status(self):
color = curses.color_pair(5)
if self.status == CheckFinalStatus.COMPLIANT: color = curses.color_pair(2)
elif self.status == CheckFinalStatus.BLOCKED: color = curses.color_pair(3)
stat_str = str(self.status.value if hasattr(self.status, "value") else self.status)
self.stdscr.addstr(18, 3, f"FINAL STATUS: {stat_str.upper()}", color | curses.A_BOLD)
if self.status == CheckFinalStatus.COMPLIANT:
color = curses.color_pair(2)
elif self.status == CheckFinalStatus.BLOCKED:
color = curses.color_pair(3)
stat_str = str(
self.status.value if hasattr(self.status, "value") else self.status
)
self.stdscr.addstr(
18, 3, f"FINAL STATUS: {stat_str.upper()}", color | curses.A_BOLD
)
if self.report_id:
self.stdscr.addstr(19, 3, f"Report ID: {self.report_id}")
@@ -413,22 +474,36 @@ class CleanReleaseTUI:
self.stdscr.addstr(20, 32, f"Publication: {publication.status}")
if self.violations_list:
self.stdscr.addstr(21, 3, f"Violations Details ({len(self.violations_list)} total):", curses.color_pair(3) | curses.A_BOLD)
self.stdscr.addstr(
21,
3,
f"Violations Details ({len(self.violations_list)} total):",
curses.color_pair(3) | curses.A_BOLD,
)
row = 22
for i, v in enumerate(self.violations_list[:5]):
v_cat = str(getattr(v, "code", "VIOLATION"))
msg = str(getattr(v, "message", "Violation detected"))
location = str(
getattr(v, "artifact_path", "")
or getattr(getattr(v, "evidence_json", {}), "get", lambda *_: "")("location", "")
or getattr(getattr(v, "evidence_json", {}), "get", lambda *_: "")(
"location", ""
)
)
msg_text = f"[{v_cat}] {msg} (Loc: {location})"
self.stdscr.addstr(row + i, 5, msg_text[:70], curses.color_pair(3))
if self.last_error:
self.stdscr.addstr(27, 3, f"Error: {self.last_error}"[:100], curses.color_pair(3) | curses.A_BOLD)
self.stdscr.addstr(
27,
3,
f"Error: {self.last_error}"[:100],
curses.color_pair(3) | curses.A_BOLD,
)
def draw_footer(self, max_y: int, max_x: int):
footer_text = " F5 Run F6 Manifest F7 Refresh F8 Approve F9 Publish F10 Exit ".center(max_x)
footer_text = " F5 Run F6 Manifest F7 Refresh F8 Approve F9 Publish F10 Exit ".center(
max_x
)
self.stdscr.attron(curses.color_pair(1))
self.stdscr.addstr(max_y - 1, 0, footer_text[:max_x])
self.stdscr.attroff(curses.color_pair(1))
@@ -446,7 +521,9 @@ class CleanReleaseTUI:
self.refresh_screen()
try:
result = self.facade.run_compliance(candidate_id=self.candidate_id, actor="operator")
result = self.facade.run_compliance(
candidate_id=self.candidate_id, actor="operator"
)
except Exception as exc: # noqa: BLE001
self.status = CheckFinalStatus.FAILED
self.last_error = str(exc)
@@ -456,7 +533,9 @@ class CleanReleaseTUI:
self.checks_progress = [
{
"stage": stage.stage_name,
"status": CheckStageStatus.PASS if str(stage.decision).upper() == "PASSED" else CheckStageStatus.FAIL,
"status": CheckStageStatus.PASS
if str(stage.decision).upper() == "PASSED"
else CheckStageStatus.FAIL,
}
for stage in result.stage_runs
]
@@ -472,11 +551,14 @@ class CleanReleaseTUI:
self.status = CheckFinalStatus.FAILED
self.refresh_overview()
self.refresh_screen()
# [/DEF:run_checks:Function]
def build_manifest(self):
try:
manifest = self.facade.build_manifest(candidate_id=self.candidate_id, actor="operator")
manifest = self.facade.build_manifest(
candidate_id=self.candidate_id, actor="operator"
)
self.status = "READY"
self.report_id = None
self.violations_list = []
@@ -570,11 +652,13 @@ class CleanReleaseTUI:
self.approve_latest()
elif char == curses.KEY_F9:
self.publish_latest()
# [/DEF:CleanReleaseTUI:Class]
def tui_main(stdscr: curses.window):
curses.curs_set(0) # Hide cursor
curses.curs_set(0) # Hide cursor
app = CleanReleaseTUI(stdscr)
app.loop()
@@ -597,4 +681,4 @@ def main() -> int:
if __name__ == "__main__":
sys.exit(main())
# [/DEF:backend.src.scripts.clean_release_tui:Module]
# [/DEF:CleanReleaseTuiScript:Module]

19
backend/src/scripts/create_admin.py
View File

@@ -1,12 +1,12 @@
# [DEF:backend.src.scripts.create_admin:Module]
# [DEF:CreateAdminScript:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: admin, setup, user, auth, cli
# @PURPOSE: CLI tool for creating the initial admin user.
# @LAYER: Scripts
# @RELATION: USES -> backend.src.core.auth.security
# @RELATION: USES -> backend.src.core.database
# @RELATION: USES -> backend.src.models.auth
# @RELATION: USES -> [AuthSecurityModule]
# @RELATION: USES -> [DatabaseModule]
# @RELATION: USES -> [AuthModels]
#
# @INVARIANT: Admin user must have the "Admin" role.
@@ -24,6 +24,7 @@ from src.models.auth import User, Role
from src.core.logger import logger, belief_scope
# [/SECTION]
# [DEF:create_admin:Function]
# @PURPOSE: Creates an admin user and necessary roles/permissions.
# @PRE: username and password provided via CLI.
@@ -36,7 +37,9 @@ def create_admin(username, password, email=None):
with belief_scope("create_admin"):
db = AuthSessionLocal()
try:
normalized_email = email.strip() if isinstance(email, str) and email.strip() else None
normalized_email = (
email.strip() if isinstance(email, str) and email.strip() else None
)
# 1. Ensure Admin role exists
admin_role = db.query(Role).filter(Role.name == "Admin").first()
@@ -60,7 +63,7 @@ def create_admin(username, password, email=None):
email=normalized_email,
password_hash=get_password_hash(password),
auth_source="LOCAL",
is_active=True
is_active=True,
)
new_user.roles.append(admin_role)
db.add(new_user)
@@ -74,6 +77,8 @@ def create_admin(username, password, email=None):
raise
finally:
db.close()
# [/DEF:create_admin:Function]
if __name__ == "__main__":
@@ -91,4 +96,4 @@ if __name__ == "__main__":
except Exception:
sys.exit(1)
# [/DEF:backend.src.scripts.create_admin:Module]
# [/DEF:CreateAdminScript:Module]

8
backend/src/scripts/dataset_dashboard_analysis.json
View File

@@ -10,7 +10,7 @@
},
"changed_by_name": "Superset Admin",
"changed_on": "2026-02-24T19:24:01.850617",
"changed_on_delta_humanized": "20 days ago",
"changed_on_delta_humanized": "29 days ago",
"charts": [
"TA-0001-001 test_chart"
],
@@ -19,7 +19,7 @@
"id": 1,
"last_name": "Admin"
},
"created_on_delta_humanized": "26 days ago",
"created_on_delta_humanized": "a month ago",
"css": null,
"dashboard_title": "TA-0001 Test dashboard",
"id": 13,
@@ -54,7 +54,7 @@
"last_name": "Admin"
},
"changed_on": "2026-02-18T14:56:04.863722",
"changed_on_humanized": "26 days ago",
"changed_on_humanized": "a month ago",
"column_formats": {},
"columns": [
{
@@ -424,7 +424,7 @@
"last_name": "Admin"
},
"created_on": "2026-02-18T14:56:04.317950",
"created_on_humanized": "26 days ago",
"created_on_humanized": "a month ago",
"database": {
"allow_multi_catalog": false,
"backend": "postgresql",

65
backend/src/scripts/migrate_sqlite_to_postgres.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.scripts.migrate_sqlite_to_postgres:Module]
# [DEF:MigrateSqliteToPostgresScript:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: migration, sqlite, postgresql, config, task_logs, task_records
@@ -30,7 +30,10 @@ from src.core.logger import belief_scope, logger
# [DEF:Constants:Section]
DEFAULT_TARGET_URL = os.getenv(
"DATABASE_URL",
os.getenv("POSTGRES_URL", "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools"),
os.getenv(
"POSTGRES_URL",
"postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools",
),
)
# [/DEF:Constants:Section]
@@ -56,6 +59,8 @@ def _json_load_if_needed(value: Any) -> Any:
except json.JSONDecodeError:
return value
return value
# [/DEF:_json_load_if_needed:Function]
@@ -75,6 +80,8 @@ def _find_legacy_config_path(explicit_path: Optional[str]) -> Optional[Path]:
if candidate.exists():
return candidate
return None
# [/DEF:_find_legacy_config_path:Function]
@@ -85,6 +92,8 @@ def _connect_sqlite(path: Path) -> sqlite3.Connection:
conn = sqlite3.connect(str(path))
conn.row_factory = sqlite3.Row
return conn
# [/DEF:_connect_sqlite:Function]
@@ -150,6 +159,8 @@ def _ensure_target_schema(engine) -> None:
with engine.begin() as conn:
for stmt in stmts:
conn.execute(text(stmt))
# [/DEF:_ensure_target_schema:Function]
@@ -158,7 +169,9 @@ def _ensure_target_schema(engine) -> None:
def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:
with belief_scope("_migrate_config"):
if legacy_config_path is None:
logger.info("[_migrate_config][Action] No legacy config.json found, skipping")
logger.info(
"[_migrate_config][Action] No legacy config.json found, skipping"
)
return 0
payload = json.loads(legacy_config_path.read_text(encoding="utf-8"))
@@ -174,8 +187,13 @@ def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:
),
{"payload": json.dumps(payload, ensure_ascii=True)},
)
logger.info("[_migrate_config][Coherence:OK] Config migrated from %s", legacy_config_path)
logger.info(
"[_migrate_config][Coherence:OK] Config migrated from %s",
legacy_config_path,
)
return 1
# [/DEF:_migrate_config:Function]
@@ -183,7 +201,12 @@ def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:
# @PURPOSE: Migrates task_records and task_logs from SQLite into PostgreSQL.
def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str, int]:
with belief_scope("_migrate_tasks_and_logs"):
stats = {"task_records_total": 0, "task_records_inserted": 0, "task_logs_total": 0, "task_logs_inserted": 0}
stats = {
"task_records_total": 0,
"task_records_inserted": 0,
"task_logs_total": 0,
"task_logs_inserted": 0,
}
rows = sqlite_conn.execute(
"""
@@ -228,11 +251,17 @@ def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str
"environment_id": environment_id,
"started_at": row["started_at"],
"finished_at": row["finished_at"],
"logs": json.dumps(logs_obj, ensure_ascii=True) if logs_obj is not None else None,
"logs": json.dumps(logs_obj, ensure_ascii=True)
if logs_obj is not None
else None,
"error": row["error"],
"result": json.dumps(result_obj, ensure_ascii=True) if result_obj is not None else None,
"result": json.dumps(result_obj, ensure_ascii=True)
if result_obj is not None
else None,
"created_at": row["created_at"],
"params": json.dumps(params_obj, ensure_ascii=True) if params_obj is not None else None,
"params": json.dumps(params_obj, ensure_ascii=True)
if params_obj is not None
else None,
},
)
if res.rowcount and res.rowcount > 0:
@@ -292,14 +321,20 @@ def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str
stats["task_logs_total"],
)
return stats
# [/DEF:_migrate_tasks_and_logs:Function]
# [DEF:run_migration:Function]
# @PURPOSE: Orchestrates migration from SQLite/file to PostgreSQL.
def run_migration(sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]) -> Dict[str, int]:
def run_migration(
sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]
) -> Dict[str, int]:
with belief_scope("run_migration"):
logger.info("[run_migration][Entry] sqlite=%s target=%s", sqlite_path, target_url)
logger.info(
"[run_migration][Entry] sqlite=%s target=%s", sqlite_path, target_url
)
if not sqlite_path.exists():
raise FileNotFoundError(f"SQLite source not found: {sqlite_path}")
@@ -313,6 +348,8 @@ def run_migration(sqlite_path: Path, target_url: str, legacy_config_path: Option
return stats
finally:
sqlite_conn.close()
# [/DEF:run_migration:Function]
@@ -344,7 +381,11 @@ def main() -> int:
sqlite_path = Path(args.sqlite_path)
legacy_config_path = _find_legacy_config_path(args.config_path)
try:
stats = run_migration(sqlite_path=sqlite_path, target_url=args.target_url, legacy_config_path=legacy_config_path)
stats = run_migration(
sqlite_path=sqlite_path,
target_url=args.target_url,
legacy_config_path=legacy_config_path,
)
print("Migration completed.")
print(json.dumps(stats, indent=2))
return 0
@@ -358,4 +399,4 @@ if __name__ == "__main__":
raise SystemExit(main())
# [/DEF:main:Function]
# [/DEF:backend.src.scripts.migrate_sqlite_to_postgres:Module]
# [/DEF:MigrateSqliteToPostgresScript:Module]

157
backend/src/scripts/seed_superset_load_test.py
View File

@@ -1,11 +1,11 @@
# [DEF:backend.src.scripts.seed_superset_load_test:Module]
# [DEF:SeedSupersetLoadTestScript:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: superset, load-test, charts, dashboards, seed, stress
# @PURPOSE: Creates randomized load-test data in Superset by cloning chart configurations and creating dashboards in target environments.
# @LAYER: Scripts
# @RELATION: USES -> backend.src.core.config_manager.ConfigManager
# @RELATION: USES -> backend.src.core.superset_client.SupersetClient
# @RELATION: USES -> [ConfigManager]
# @RELATION: USES -> [SupersetClient]
# @INVARIANT: Created chart and dashboard names are globally unique for one script run.
# [SECTION: IMPORTS]
@@ -31,15 +31,42 @@ from src.core.superset_client import SupersetClient
# @PRE: Script is called from CLI.
# @POST: Returns validated argument namespace.
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Seed Superset with load-test charts and dashboards")
parser.add_argument("--envs", nargs="+", default=["ss1", "ss2"], help="Target environment IDs")
parser.add_argument("--charts", type=int, default=10000, help="Target number of charts to create")
parser.add_argument("--dashboards", type=int, default=500, help="Target number of dashboards to create")
parser.add_argument("--template-pool-size", type=int, default=200, help="How many source charts to sample as templates per env")
parser.add_argument("--seed", type=int, default=None, help="Optional RNG seed for reproducibility")
parser.add_argument("--max-errors", type=int, default=100, help="Stop early if errors exceed this threshold")
parser.add_argument("--dry-run", action="store_true", help="Do not write data, only validate setup")
parser = argparse.ArgumentParser(
description="Seed Superset with load-test charts and dashboards"
)
parser.add_argument(
"--envs", nargs="+", default=["ss1", "ss2"], help="Target environment IDs"
)
parser.add_argument(
"--charts", type=int, default=10000, help="Target number of charts to create"
)
parser.add_argument(
"--dashboards",
type=int,
default=500,
help="Target number of dashboards to create",
)
parser.add_argument(
"--template-pool-size",
type=int,
default=200,
help="How many source charts to sample as templates per env",
)
parser.add_argument(
"--seed", type=int, default=None, help="Optional RNG seed for reproducibility"
)
parser.add_argument(
"--max-errors",
type=int,
default=100,
help="Stop early if errors exceed this threshold",
)
parser.add_argument(
"--dry-run", action="store_true", help="Do not write data, only validate setup"
)
return parser.parse_args()
# [/DEF:_parse_args:Function]
@@ -52,6 +79,8 @@ def _extract_result_payload(payload: Dict) -> Dict:
if isinstance(result, dict):
return result
return payload
# [/DEF:_extract_result_payload:Function]
@@ -67,6 +96,8 @@ def _extract_created_id(payload: Dict) -> Optional[int]:
if isinstance(result, dict) and isinstance(result.get("id"), int):
return int(result["id"])
return None
# [/DEF:_extract_created_id:Function]
@@ -75,14 +106,38 @@ def _extract_created_id(payload: Dict) -> Optional[int]:
# @PRE: used_names is mutable set for collision tracking.
# @POST: Returns a unique string and stores it in used_names.
def _generate_unique_name(prefix: str, used_names: set[str], rng: random.Random) -> str:
adjectives = ["amber", "rapid", "frozen", "delta", "lunar", "vector", "cobalt", "silent", "neon", "solar"]
nouns = ["falcon", "matrix", "signal", "harbor", "stream", "vertex", "bridge", "orbit", "pulse", "forge"]
adjectives = [
"amber",
"rapid",
"frozen",
"delta",
"lunar",
"vector",
"cobalt",
"silent",
"neon",
"solar",
]
nouns = [
"falcon",
"matrix",
"signal",
"harbor",
"stream",
"vertex",
"bridge",
"orbit",
"pulse",
"forge",
]
while True:
token = uuid.uuid4().hex[:8]
candidate = f"{prefix}_{rng.choice(adjectives)}_{rng.choice(nouns)}_{rng.randint(100, 999)}_{token}"
if candidate not in used_names:
used_names.add(candidate)
return candidate
# [/DEF:_generate_unique_name:Function]
@@ -106,7 +161,9 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:
env = Environment(**row)
configured[env.id] = env
except Exception as exc:
logger.warning(f"[REFLECT] Failed loading environments from {config_path}: {exc}")
logger.warning(
f"[REFLECT] Failed loading environments from {config_path}: {exc}"
)
for env_id in env_ids:
env = configured.get(env_id)
@@ -115,6 +172,8 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:
resolved[env_id] = env
return resolved
# [/DEF:_resolve_target_envs:Function]
@@ -122,11 +181,21 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:
# @PURPOSE: Builds a pool of source chart templates to clone in one environment.
# @PRE: Client is authenticated.
# @POST: Returns non-empty list of chart payload templates.
def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: random.Random) -> List[Dict]:
def _build_chart_template_pool(
client: SupersetClient, pool_size: int, rng: random.Random
) -> List[Dict]:
list_query = {
"page": 0,
"page_size": 1000,
"columns": ["id", "slice_name", "datasource_id", "datasource_type", "viz_type", "params", "query_context"],
"columns": [
"id",
"slice_name",
"datasource_id",
"datasource_type",
"viz_type",
"params",
"query_context",
],
}
rows = client.network.fetch_paginated_data(
endpoint="/chart/",
@@ -137,7 +206,11 @@ def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: rand
if not candidates:
raise RuntimeError("No source charts available for templating")
selected = candidates if len(candidates) <= pool_size else rng.sample(candidates, pool_size)
selected = (
candidates
if len(candidates) <= pool_size
else rng.sample(candidates, pool_size)
)
templates: List[Dict] = []
for row in selected:
@@ -146,7 +219,9 @@ def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: rand
detail = _extract_result_payload(detail_payload)
datasource_id = detail.get("datasource_id")
datasource_type = detail.get("datasource_type") or row.get("datasource_type") or "table"
datasource_type = (
detail.get("datasource_type") or row.get("datasource_type") or "table"
)
if datasource_id is None:
continue
@@ -172,6 +247,8 @@ def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: rand
raise RuntimeError("Could not build templates with datasource metadata")
return templates
# [/DEF:_build_chart_template_pool:Function]
@@ -195,23 +272,33 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
client = SupersetClient(env)
client.authenticate()
clients[env_id] = client
templates_by_env[env_id] = _build_chart_template_pool(client, args.template_pool_size, rng)
logger.info(f"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates")
templates_by_env[env_id] = _build_chart_template_pool(
client, args.template_pool_size, rng
)
logger.info(
f"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates"
)
errors = 0
env_ids = list(env_map.keys())
for idx in range(args.dashboards):
env_id = env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)
env_id = (
env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)
)
dashboard_title = _generate_unique_name("lt_dash", used_dashboard_names, rng)
if args.dry_run:
logger.info(f"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}")
logger.info(
f"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}"
)
continue
try:
payload = {"dashboard_title": dashboard_title, "published": False}
created = clients[env_id].network.request("POST", "/dashboard/", data=json.dumps(payload))
created = clients[env_id].network.request(
"POST", "/dashboard/", data=json.dumps(payload)
)
dashboard_id = _extract_created_id(created)
if dashboard_id is None:
raise RuntimeError(f"Dashboard create response missing id: {created}")
@@ -220,7 +307,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
errors += 1
logger.error(f"[EXPLORE] Failed creating dashboard in {env_id}: {exc}")
if errors >= args.max_errors:
raise RuntimeError(f"Stopping due to max errors reached ({errors})") from exc
raise RuntimeError(
f"Stopping due to max errors reached ({errors})"
) from exc
if args.dry_run:
return {
@@ -232,7 +321,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
for env_id in env_ids:
if not created_dashboards[env_id]:
raise RuntimeError(f"No dashboards created in environment {env_id}; cannot bind charts")
raise RuntimeError(
f"No dashboards created in environment {env_id}; cannot bind charts"
)
for index in range(args.charts):
env_id = rng.choice(env_ids)
@@ -255,7 +346,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
payload["query_context"] = template["query_context"]
try:
created = client.network.request("POST", "/chart/", data=json.dumps(payload))
created = client.network.request(
"POST", "/chart/", data=json.dumps(payload)
)
chart_id = _extract_created_id(created)
if chart_id is None:
raise RuntimeError(f"Chart create response missing id: {created}")
@@ -267,7 +360,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
errors += 1
logger.error(f"[EXPLORE] Failed creating chart in {env_id}: {exc}")
if errors >= args.max_errors:
raise RuntimeError(f"Stopping due to max errors reached ({errors})") from exc
raise RuntimeError(
f"Stopping due to max errors reached ({errors})"
) from exc
return {
"dry_run": False,
@@ -277,6 +372,8 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
"total_dashboards": sum(len(ids) for ids in created_dashboards.values()),
"total_charts": sum(len(ids) for ids in created_charts.values()),
}
# [/DEF:seed_superset_load_data:Function]
@@ -288,7 +385,9 @@ def main() -> None:
with belief_scope("seed_superset_load_test.main"):
args = _parse_args()
result = seed_superset_load_data(args)
logger.info(f"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}")
logger.info(
f"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}"
)
# [/DEF:main:Function]
@@ -297,4 +396,4 @@ def main() -> None:
if __name__ == "__main__":
main()
# [/DEF:backend.src.scripts.seed_superset_load_test:Module]
# [/DEF:SeedSupersetLoadTestScript:Module]

22
backend/src/services/auth_service.py
View File

@@ -3,11 +3,11 @@
# @SEMANTICS: auth, service, business-logic, login, jwt, adfs, jit-provisioning
# @PURPOSE: Orchestrates credential authentication and ADFS JIT user provisioning.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [AuthRepository]
# @RELATION: DEPENDS_ON -> [verify_password]
# @RELATION: DEPENDS_ON -> [create_access_token]
# @RELATION: DEPENDS_ON -> [User]
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: [DEPENDS_ON] ->[AuthRepository]
# @RELATION: [DEPENDS_ON] ->[verify_password]
# @RELATION: [DEPENDS_ON] ->[create_access_token]
# @RELATION: [DEPENDS_ON] ->[User]
# @RELATION: [DEPENDS_ON] ->[Role]
# @INVARIANT: Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles.
# @PRE: Core auth models and security utilities available.
# @POST: User identity verified and session tokens issued according to role scopes.
@@ -29,6 +29,9 @@ from ..core.logger import belief_scope
# [DEF:AuthService:Class]
# @COMPLEXITY: 3
# @PURPOSE: Provides high-level authentication services.
# @RELATION: [DEPENDS_ON] ->[AuthRepository]
# @RELATION: [DEPENDS_ON] ->[User]
# @RELATION: [DEPENDS_ON] ->[Role]
class AuthService:
# [DEF:AuthService_init:Function]
# @COMPLEXITY: 1
@@ -51,6 +54,9 @@ class AuthService:
# @POST: Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None.
# @SIDE_EFFECT: Persists last_login update for successful authentications via repository.
# @DATA_CONTRACT: Input(str username, str password) -> Output(User | None)
# @RELATION: [DEPENDS_ON] ->[AuthRepository]
# @RELATION: [CALLS] ->[verify_password]
# @RELATION: [DEPENDS_ON] ->[User]
# @PARAM: username (str) - The username.
# @PARAM: password (str) - The plain password.
# @RETURN: Optional[User] - The authenticated user or None.
@@ -79,6 +85,9 @@ class AuthService:
# @POST: Returns session dict with non-empty access_token and token_type='bearer'.
# @SIDE_EFFECT: Generates signed JWT via auth JWT provider.
# @DATA_CONTRACT: Input(User) -> Output(Dict[str, str]{access_token, token_type})
# @RELATION: [CALLS] ->[create_access_token]
# @RELATION: [DEPENDS_ON] ->[User]
# @RELATION: [DEPENDS_ON] ->[Role]
# @PARAM: user (User) - The authenticated user.
# @RETURN: Dict[str, str] - Session data.
def create_session(self, user: User) -> Dict[str, str]:
@@ -98,6 +107,9 @@ class AuthService:
# @POST: Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state.
# @SIDE_EFFECT: May insert new User, mutate user.roles, commit transaction, and refresh ORM state.
# @DATA_CONTRACT: Input(Dict[str, Any]{upn|email, email, groups[]}) -> Output(User persisted)
# @RELATION: [DEPENDS_ON] ->[AuthRepository]
# @RELATION: [DEPENDS_ON] ->[User]
# @RELATION: [DEPENDS_ON] ->[Role]
# @PARAM: user_info (Dict[str, Any]) - Claims from ADFS token.
# @RETURN: User - The provisioned user.
def provision_adfs_user(self, user_info: Dict[str, Any]) -> User:

10
backend/src/services/clean_release/__init__.py
View File

@@ -1,7 +1,11 @@
# [DEF:clean_release:Module]
# [DEF:CleanReleaseContracts:Module]
# @COMPLEXITY: 3
# @PURPOSE: Redesigned clean release compliance subsystem.
# @PURPOSE: Publish the canonical semantic root for the clean-release backend service cluster.
# @LAYER: Domain
# @RELATION: [DEPENDS_ON] ->[ComplianceOrchestrator]
# @RELATION: [DEPENDS_ON] ->[ManifestBuilder]
# @RELATION: [DEPENDS_ON] ->[PolicyEngine]
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
from ...core.logger import logger
@@ -13,4 +17,4 @@ __all__ = [
"logger",
]
# [/DEF:clean_release:Module]
# [/DEF:CleanReleaseContracts:Module]

2
backend/src/services/clean_release/__tests__/test_audit_service.py
View File

@@ -1,5 +1,5 @@
# [DEF:TestAuditService:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION: [DEPENDS_ON] ->[AuditService]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, audit, logging
# @PURPOSE: Validate audit hooks emit expected log patterns for clean release lifecycle.

76
backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py
View File

@@ -1,5 +1,5 @@
# [DEF:TestComplianceOrchestrator:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION: [DEPENDS_ON] ->[ComplianceOrchestrator]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, orchestrator, stage-state-machine
# @PURPOSE: Validate compliance orchestrator stage transitions and final status derivation.
@@ -16,7 +16,9 @@ from src.models.clean_release import (
CheckStageResult,
CheckStageStatus,
)
from src.services.clean_release.compliance_orchestrator import CleanComplianceOrchestrator
from src.services.clean_release.compliance_orchestrator import (
CleanComplianceOrchestrator,
)
from src.services.clean_release.report_builder import ComplianceReportBuilder
from src.services.clean_release.repository import CleanReleaseRepository
@@ -37,15 +39,33 @@ def test_orchestrator_stage_failure_blocks_release():
run = orchestrator.execute_stages(
run,
forced_results=[
CheckStageResult(stage=CheckStageName.DATA_PURITY, status=CheckStageStatus.PASS, details="ok"),
CheckStageResult(stage=CheckStageName.INTERNAL_SOURCES_ONLY, status=CheckStageStatus.PASS, details="ok"),
CheckStageResult(stage=CheckStageName.NO_EXTERNAL_ENDPOINTS, status=CheckStageStatus.FAIL, details="external"),
CheckStageResult(stage=CheckStageName.MANIFEST_CONSISTENCY, status=CheckStageStatus.PASS, details="ok"),
CheckStageResult(
stage=CheckStageName.DATA_PURITY,
status=CheckStageStatus.PASS,
details="ok",
),
CheckStageResult(
stage=CheckStageName.INTERNAL_SOURCES_ONLY,
status=CheckStageStatus.PASS,
details="ok",
),
CheckStageResult(
stage=CheckStageName.NO_EXTERNAL_ENDPOINTS,
status=CheckStageStatus.FAIL,
details="external",
),
CheckStageResult(
stage=CheckStageName.MANIFEST_CONSISTENCY,
status=CheckStageStatus.PASS,
details="ok",
),
],
)
run = orchestrator.finalize_run(run)
assert run.final_status == CheckFinalStatus.BLOCKED
# [/DEF:test_orchestrator_stage_failure_blocks_release:Function]
@@ -65,15 +85,33 @@ def test_orchestrator_compliant_candidate():
run = orchestrator.execute_stages(
run,
forced_results=[
CheckStageResult(stage=CheckStageName.DATA_PURITY, status=CheckStageStatus.PASS, details="ok"),
CheckStageResult(stage=CheckStageName.INTERNAL_SOURCES_ONLY, status=CheckStageStatus.PASS, details="ok"),
CheckStageResult(stage=CheckStageName.NO_EXTERNAL_ENDPOINTS, status=CheckStageStatus.PASS, details="ok"),
CheckStageResult(stage=CheckStageName.MANIFEST_CONSISTENCY, status=CheckStageStatus.PASS, details="ok"),
CheckStageResult(
stage=CheckStageName.DATA_PURITY,
status=CheckStageStatus.PASS,
details="ok",
),
CheckStageResult(
stage=CheckStageName.INTERNAL_SOURCES_ONLY,
status=CheckStageStatus.PASS,
details="ok",
),
CheckStageResult(
stage=CheckStageName.NO_EXTERNAL_ENDPOINTS,
status=CheckStageStatus.PASS,
details="ok",
),
CheckStageResult(
stage=CheckStageName.MANIFEST_CONSISTENCY,
status=CheckStageStatus.PASS,
details="ok",
),
],
)
run = orchestrator.finalize_run(run)
assert run.final_status == CheckFinalStatus.COMPLIANT
# [/DEF:test_orchestrator_compliant_candidate:Function]
@@ -87,11 +125,19 @@ def test_orchestrator_missing_stage_result():
run = orchestrator.start_check_run("cand-1", "pol-1", "tester", "tui")
run = orchestrator.execute_stages(
run,
forced_results=[CheckStageResult(stage=CheckStageName.DATA_PURITY, status=CheckStageStatus.PASS, details="ok")],
forced_results=[
CheckStageResult(
stage=CheckStageName.DATA_PURITY,
status=CheckStageStatus.PASS,
details="ok",
)
],
)
run = orchestrator.finalize_run(run)
assert run.final_status == CheckFinalStatus.FAILED
# [/DEF:test_orchestrator_missing_stage_result:Function]
@@ -106,11 +152,17 @@ def test_orchestrator_report_generation_error():
run = orchestrator.finalize_run(run)
assert run.final_status == CheckFinalStatus.FAILED
with patch.object(ComplianceReportBuilder, "build_report_payload", side_effect=ValueError("Report error")):
with patch.object(
ComplianceReportBuilder,
"build_report_payload",
side_effect=ValueError("Report error"),
):
builder = ComplianceReportBuilder(repository)
with pytest.raises(ValueError, match="Report error"):
builder.build_report_payload(run, [])
assert run.final_status == CheckFinalStatus.FAILED
# [/DEF:test_orchestrator_report_generation_error:Function]
# [/DEF:TestComplianceOrchestrator:Module]

20
backend/src/services/clean_release/__tests__/test_manifest_builder.py
View File

@@ -3,7 +3,7 @@
# @SEMANTICS: tests, clean-release, manifest, deterministic
# @PURPOSE: Validate deterministic manifest generation behavior for US1.
# @LAYER: Domain
# @RELATION: VERIFIES -> backend.src.services.clean_release.manifest_builder
# @RELATION: [DEPENDS_ON] ->[ManifestBuilder]
# @INVARIANT: Same input artifacts produce identical deterministic hash.
from src.services.clean_release.manifest_builder import build_distribution_manifest
@@ -16,8 +16,18 @@ from src.services.clean_release.manifest_builder import build_distribution_manif
# @POST: Hash and summary remain identical.
def test_manifest_deterministic_hash_for_same_input():
artifacts = [
{"path": "a.yaml", "category": "system-init", "classification": "required-system", "reason": "required"},
{"path": "b.yaml", "category": "test-data", "classification": "excluded-prohibited", "reason": "prohibited"},
{
"path": "a.yaml",
"category": "system-init",
"classification": "required-system",
"reason": "required",
},
{
"path": "b.yaml",
"category": "test-data",
"classification": "excluded-prohibited",
"reason": "prohibited",
},
]
manifest1 = build_distribution_manifest(
@@ -38,5 +48,7 @@ def test_manifest_deterministic_hash_for_same_input():
assert manifest1.deterministic_hash == manifest2.deterministic_hash
assert manifest1.summary.included_count == manifest2.summary.included_count
assert manifest1.summary.excluded_count == manifest2.summary.excluded_count
# [/DEF:test_manifest_deterministic_hash_for_same_input:Function]
# [/DEF:TestManifestBuilder:Module]
# [/DEF:TestManifestBuilder:Module]

85
backend/src/services/clean_release/__tests__/test_policy_engine.py
View File

@@ -1,19 +1,20 @@
# [DEF:__tests__/test_policy_engine:Module]
# @RELATION: VERIFIES -> ../policy_engine.py
# [DEF:TestPolicyEngine:Module]
# @RELATION: [DEPENDS_ON] ->[PolicyEngine]
# @PURPOSE: Contract testing for CleanPolicyEngine
# [/DEF:__tests__/test_policy_engine:Module]
# [/DEF:TestPolicyEngine:Module]
import pytest
from datetime import datetime
from src.models.clean_release import (
CleanProfilePolicy,
ResourceSourceRegistry,
ResourceSourceEntry,
ProfileType,
RegistryStatus
CleanProfilePolicy,
ResourceSourceRegistry,
ResourceSourceEntry,
ProfileType,
RegistryStatus,
)
from src.services.clean_release.policy_engine import CleanPolicyEngine
# @TEST_FIXTURE: policy_enterprise_clean
@pytest.fixture
def enterprise_clean_setup():
@@ -25,23 +26,30 @@ def enterprise_clean_setup():
required_system_categories=["core"],
internal_source_registry_ref="REG-1",
effective_from=datetime.now(),
profile=ProfileType.ENTERPRISE_CLEAN
profile=ProfileType.ENTERPRISE_CLEAN,
)
registry = ResourceSourceRegistry(
registry_id="REG-1",
name="Internal Registry",
entries=[
ResourceSourceEntry(source_id="S1", host="internal.com", protocol="https", purpose="p1", enabled=True)
ResourceSourceEntry(
source_id="S1",
host="internal.com",
protocol="https",
purpose="p1",
enabled=True,
)
],
updated_at=datetime.now(),
updated_by="admin",
status=RegistryStatus.ACTIVE
status=RegistryStatus.ACTIVE,
)
return policy, registry
# @TEST_SCENARIO: policy_valid
# [DEF:test_policy_valid:Function]
# @RELATION: BINDS_TO -> __tests__/test_policy_engine
# @RELATION: BINDS_TO -> TestPolicyEngine
def test_policy_valid(enterprise_clean_setup):
policy, registry = enterprise_clean_setup
engine = CleanPolicyEngine(policy, registry)
@@ -49,11 +57,13 @@ def test_policy_valid(enterprise_clean_setup):
assert result.ok is True
assert not result.blocking_reasons
# @TEST_EDGE: missing_registry_ref
# [/DEF:test_policy_valid:Function]
# [DEF:test_missing_registry_ref:Function]
# @RELATION: BINDS_TO -> __tests__/test_policy_engine
# @RELATION: BINDS_TO -> TestPolicyEngine
def test_missing_registry_ref(enterprise_clean_setup):
policy, registry = enterprise_clean_setup
policy.internal_source_registry_ref = " "
@@ -62,76 +72,95 @@ def test_missing_registry_ref(enterprise_clean_setup):
assert result.ok is False
assert "Policy missing internal_source_registry_ref" in result.blocking_reasons
# @TEST_EDGE: conflicting_registry
# [/DEF:test_missing_registry_ref:Function]
# [DEF:test_conflicting_registry:Function]
# @RELATION: BINDS_TO -> __tests__/test_policy_engine
# @RELATION: BINDS_TO -> TestPolicyEngine
def test_conflicting_registry(enterprise_clean_setup):
policy, registry = enterprise_clean_setup
registry.registry_id = "WRONG-REG"
engine = CleanPolicyEngine(policy, registry)
result = engine.validate_policy()
assert result.ok is False
assert "Policy registry ref does not match provided registry" in result.blocking_reasons
assert (
"Policy registry ref does not match provided registry"
in result.blocking_reasons
)
# @TEST_INVARIANT: deterministic_classification
# [/DEF:test_conflicting_registry:Function]
# [DEF:test_classify_artifact:Function]
# @RELATION: BINDS_TO -> __tests__/test_policy_engine
# @RELATION: BINDS_TO -> TestPolicyEngine
def test_classify_artifact(enterprise_clean_setup):
policy, registry = enterprise_clean_setup
engine = CleanPolicyEngine(policy, registry)
# Required
assert engine.classify_artifact({"category": "core", "path": "p1"}) == "required-system"
assert (
engine.classify_artifact({"category": "core", "path": "p1"})
== "required-system"
)
# Prohibited
assert engine.classify_artifact({"category": "demo", "path": "p2"}) == "excluded-prohibited"
assert (
engine.classify_artifact({"category": "demo", "path": "p2"})
== "excluded-prohibited"
)
# Allowed
assert engine.classify_artifact({"category": "others", "path": "p3"}) == "allowed"
# @TEST_EDGE: external_endpoint
# [/DEF:test_classify_artifact:Function]
# [DEF:test_validate_resource_source:Function]
# @RELATION: BINDS_TO -> __tests__/test_policy_engine
# @RELATION: BINDS_TO -> TestPolicyEngine
def test_validate_resource_source(enterprise_clean_setup):
policy, registry = enterprise_clean_setup
engine = CleanPolicyEngine(policy, registry)
# Internal (OK)
res_ok = engine.validate_resource_source("internal.com")
assert res_ok.ok is True
# External (Blocked)
res_fail = engine.validate_resource_source("external.evil")
assert res_fail.ok is False
assert res_fail.violation["category"] == "external-source"
assert res_fail.violation["blocked_release"] is True
# [/DEF:test_validate_resource_source:Function]
# [DEF:test_evaluate_candidate:Function]
# @RELATION: BINDS_TO -> __tests__/test_policy_engine
# @RELATION: BINDS_TO -> TestPolicyEngine
def test_evaluate_candidate(enterprise_clean_setup):
policy, registry = enterprise_clean_setup
engine = CleanPolicyEngine(policy, registry)
artifacts = [
{"path": "core.js", "category": "core"},
{"path": "demo.sql", "category": "demo"}
{"path": "demo.sql", "category": "demo"},
]
sources = ["internal.com", "google.com"]
classified, violations = engine.evaluate_candidate(artifacts, sources)
assert len(classified) == 2
assert classified[0]["classification"] == "required-system"
assert classified[1]["classification"] == "excluded-prohibited"
# 1 violation for demo artifact + 1 for google.com source
assert len(violations) == 2
assert violations[0]["category"] == "data-purity"
assert violations[1]["category"] == "external-source"
# [/DEF:test_evaluate_candidate:Function]

2
backend/src/services/clean_release/__tests__/test_preparation_service.py
View File

@@ -3,7 +3,7 @@
# @SEMANTICS: tests, clean-release, preparation, flow
# @PURPOSE: Validate release candidate preparation flow, including policy evaluation and manifest persisting.
# @LAYER: Domain
# @RELATION: [DEPENDS_ON] ->[backend.src.services.clean_release.preparation_service:Module]
# @RELATION: [DEPENDS_ON] ->[PreparationService]
# @INVARIANT: Candidate preparation always persists manifest and candidate status deterministically.
import pytest

18
backend/src/services/clean_release/__tests__/test_report_builder.py
View File

@@ -1,5 +1,5 @@
# [DEF:TestReportBuilder:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION: [DEPENDS_ON] ->[ReportBuilder]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, report-builder, counters
# @PURPOSE: Validate compliance report builder counter integrity and blocked-run constraints.
@@ -37,6 +37,8 @@ def _terminal_run(status: CheckFinalStatus) -> ComplianceCheckRun:
execution_mode=ExecutionMode.TUI,
checks=[],
)
# [/DEF:_terminal_run:Function]
@@ -54,6 +56,8 @@ def _blocking_violation() -> ComplianceViolation:
blocked_release=True,
detected_at=datetime.now(timezone.utc),
)
# [/DEF:_blocking_violation:Function]
@@ -66,6 +70,8 @@ def test_report_builder_blocked_requires_blocking_violations():
with pytest.raises(ValueError):
builder.build_report_payload(run, [])
# [/DEF:test_report_builder_blocked_requires_blocking_violations:Function]
@@ -79,14 +85,16 @@ def test_report_builder_blocked_with_two_violations():
v2 = _blocking_violation()
v2.violation_id = "viol-2"
v2.category = ViolationCategory.DATA_PURITY
report = builder.build_report_payload(run, [v1, v2])
assert report.check_run_id == run.check_run_id
assert report.candidate_id == run.candidate_id
assert report.final_status == CheckFinalStatus.BLOCKED
assert report.violations_count == 2
assert report.blocking_violations_count == 2
# [/DEF:test_report_builder_blocked_with_two_violations:Function]
@@ -100,6 +108,8 @@ def test_report_builder_counter_consistency():
assert report.violations_count == 1
assert report.blocking_violations_count == 1
# [/DEF:test_report_builder_counter_consistency:Function]
@@ -114,5 +124,7 @@ def test_missing_operator_summary():
builder.build_report_payload(run, [])
assert "Cannot build report for non-terminal run" in str(exc.value)
# [/DEF:test_missing_operator_summary:Function]
# [/DEF:TestReportBuilder:Module]

2
backend/src/services/clean_release/__tests__/test_source_isolation.py
View File

@@ -1,5 +1,5 @@
# [DEF:TestSourceIsolation:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION: [DEPENDS_ON] ->[SourceIsolation]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, source-isolation, internal-only
# @PURPOSE: Verify internal source registry validation behavior.

2
backend/src/services/clean_release/__tests__/test_stages.py
View File

@@ -1,5 +1,5 @@
# [DEF:TestStages:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION: [DEPENDS_ON] ->[ComplianceStages]
# @COMPLEXITY: 3
# @SEMANTICS: tests, clean-release, compliance, stages
# @PURPOSE: Validate final status derivation logic from stage results.

67
backend/src/services/clean_release/approval_service.py
View File

@@ -1,11 +1,11 @@
# [DEF:approval_service:Module]
# [DEF:ApprovalService:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, approval, decision, lifecycle, gate
# @PURPOSE: Enforce approval/rejection gates over immutable compliance reports.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.audit_service
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @RELATION: [DEPENDS_ON] ->[AuditService]
# @INVARIANT: Approval is allowed only for PASSED report bound to candidate; decisions are append-only.
from __future__ import annotations
@@ -26,12 +26,16 @@ from .repository import CleanReleaseRepository
# @PURPOSE: Provide append-only in-memory storage for approval decisions.
# @PRE: repository is initialized.
# @POST: Returns mutable decision list attached to repository.
def _get_or_init_decisions_store(repository: CleanReleaseRepository) -> List[ApprovalDecision]:
def _get_or_init_decisions_store(
repository: CleanReleaseRepository,
) -> List[ApprovalDecision]:
decisions = getattr(repository, "approval_decisions", None)
if decisions is None:
decisions = []
setattr(repository, "approval_decisions", decisions)
return decisions
# [/DEF:_get_or_init_decisions_store:Function]
@@ -39,12 +43,20 @@ def _get_or_init_decisions_store(repository: CleanReleaseRepository) -> List[App
# @PURPOSE: Resolve latest approval decision for candidate from append-only store.
# @PRE: candidate_id is non-empty.
# @POST: Returns latest ApprovalDecision or None.
def _latest_decision_for_candidate(repository: CleanReleaseRepository, candidate_id: str) -> ApprovalDecision | None:
def _latest_decision_for_candidate(
repository: CleanReleaseRepository, candidate_id: str
) -> ApprovalDecision | None:
decisions = _get_or_init_decisions_store(repository)
scoped = [item for item in decisions if item.candidate_id == candidate_id]
if not scoped:
return None
return sorted(scoped, key=lambda item: item.decided_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True)[0]
return sorted(
scoped,
key=lambda item: item.decided_at or datetime.min.replace(tzinfo=timezone.utc),
reverse=True,
)[0]
# [/DEF:_latest_decision_for_candidate:Function]
@@ -70,6 +82,8 @@ def _resolve_candidate_and_report(
raise ApprovalGateError("report belongs to another candidate")
return candidate, report
# [/DEF:_resolve_candidate_and_report:Function]
@@ -86,7 +100,9 @@ def approve_candidate(
comment: str | None = None,
) -> ApprovalDecision:
with belief_scope("approval_service.approve_candidate"):
logger.reason(f"[REASON] Evaluating approve gate candidate_id={candidate_id} report_id={report_id}")
logger.reason(
f"[REASON] Evaluating approve gate candidate_id={candidate_id} report_id={report_id}"
)
if not decided_by or not decided_by.strip():
raise ApprovalGateError("decided_by must be non-empty")
@@ -101,7 +117,10 @@ def approve_candidate(
raise ApprovalGateError("approve requires PASSED compliance report")
latest = _latest_decision_for_candidate(repository, candidate_id)
if latest is not None and latest.decision == ApprovalDecisionType.APPROVED.value:
if (
latest is not None
and latest.decision == ApprovalDecisionType.APPROVED.value
):
raise ApprovalGateError("candidate is already approved")
if candidate.status == CandidateStatus.APPROVED.value:
@@ -117,7 +136,9 @@ def approve_candidate(
except ApprovalGateError:
raise
except Exception as exc: # noqa: BLE001
logger.explore(f"[EXPLORE] Candidate transition to APPROVED failed candidate_id={candidate_id}: {exc}")
logger.explore(
f"[EXPLORE] Candidate transition to APPROVED failed candidate_id={candidate_id}: {exc}"
)
raise ApprovalGateError(str(exc)) from exc
decision = ApprovalDecision(
@@ -130,9 +151,15 @@ def approve_candidate(
comment=comment,
)
_get_or_init_decisions_store(repository).append(decision)
audit_preparation(candidate_id, "APPROVED", repository=repository, actor=decided_by)
logger.reflect(f"[REFLECT] Approval persisted candidate_id={candidate_id} decision_id={decision.id}")
audit_preparation(
candidate_id, "APPROVED", repository=repository, actor=decided_by
)
logger.reflect(
f"[REFLECT] Approval persisted candidate_id={candidate_id} decision_id={decision.id}"
)
return decision
# [/DEF:approve_candidate:Function]
@@ -149,7 +176,9 @@ def reject_candidate(
comment: str | None = None,
) -> ApprovalDecision:
with belief_scope("approval_service.reject_candidate"):
logger.reason(f"[REASON] Evaluating reject decision candidate_id={candidate_id} report_id={report_id}")
logger.reason(
f"[REASON] Evaluating reject decision candidate_id={candidate_id} report_id={report_id}"
)
if not decided_by or not decided_by.strip():
raise ApprovalGateError("decided_by must be non-empty")
@@ -170,9 +199,15 @@ def reject_candidate(
comment=comment,
)
_get_or_init_decisions_store(repository).append(decision)
audit_preparation(candidate_id, "REJECTED", repository=repository, actor=decided_by)
logger.reflect(f"[REFLECT] Rejection persisted candidate_id={candidate_id} decision_id={decision.id}")
audit_preparation(
candidate_id, "REJECTED", repository=repository, actor=decided_by
)
logger.reflect(
f"[REFLECT] Rejection persisted candidate_id={candidate_id} decision_id={decision.id}"
)
return decision
# [/DEF:reject_candidate:Function]
# [/DEF:backend.src.services.clean_release.approval_service:Module]
# [/DEF:backend.src.services.clean_release.approval_service:Module]

23
backend/src/services/clean_release/artifact_catalog_loader.py
View File

@@ -1,9 +1,9 @@
# [DEF:artifact_catalog_loader:Module]
# [DEF:ArtifactCatalogLoader:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, artifacts, bootstrap, json, tui
# @PURPOSE: Load bootstrap artifact catalogs for clean release real-mode flows.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release.CandidateArtifact
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @INVARIANT: Artifact catalog must produce deterministic CandidateArtifact entries with required identity and checksum fields.
from __future__ import annotations
@@ -29,7 +29,9 @@ def load_bootstrap_artifacts(path: str, candidate_id: str) -> List[CandidateArti
payload = json.loads(catalog_path.read_text(encoding="utf-8"))
raw_artifacts = payload.get("artifacts") if isinstance(payload, dict) else payload
if not isinstance(raw_artifacts, list):
raise ValueError("artifact catalog must be a list or an object with 'artifacts' list")
raise ValueError(
"artifact catalog must be a list or an object with 'artifacts' list"
)
artifacts: List[CandidateArtifact] = []
for index, raw_artifact in enumerate(raw_artifacts, start=1):
@@ -47,9 +49,18 @@ def load_bootstrap_artifacts(path: str, candidate_id: str) -> List[CandidateArti
if not artifact_sha256:
raise ValueError(f"artifact #{index} missing required field 'sha256'")
if not isinstance(artifact_size, int) or artifact_size < 0:
raise ValueError(f"artifact #{index} field 'size' must be non-negative integer")
raise ValueError(
f"artifact #{index} field 'size' must be non-negative integer"
)
category = str(raw_artifact.get("detected_category") or raw_artifact.get("category") or "").strip() or None
category = (
str(
raw_artifact.get("detected_category")
or raw_artifact.get("category")
or ""
).strip()
or None
)
source_uri = str(raw_artifact.get("source_uri", "")).strip() or None
source_host = str(raw_artifact.get("source_host", "")).strip() or None
metadata_json = raw_artifact.get("metadata_json")
@@ -89,6 +100,8 @@ def load_bootstrap_artifacts(path: str, candidate_id: str) -> List[CandidateArti
)
return artifacts
# [/DEF:load_bootstrap_artifacts:Function]
# [/DEF:backend.src.services.clean_release.artifact_catalog_loader:Module]

28
backend/src/services/clean_release/audit_service.py
View File

@@ -1,9 +1,9 @@
# [DEF:audit_service:Module]
# [DEF:AuditService:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, audit, lifecycle, logging
# @PURPOSE: Provide lightweight audit hooks for clean release preparation/check/report lifecycle.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> backend.src.core.logger
# @RELATION: [DEPENDS_ON] ->[LoggerModule]
# @INVARIANT: Audit hooks are append-only log actions.
from __future__ import annotations
@@ -20,8 +20,12 @@ def _append_event(repository, payload: Dict[str, Any]) -> None:
repository.append_audit_event(payload)
def audit_preparation(candidate_id: str, status: str, repository=None, actor: str = "system") -> None:
logger.info(f"[REASON] clean-release preparation candidate={candidate_id} status={status}")
def audit_preparation(
candidate_id: str, status: str, repository=None, actor: str = "system"
) -> None:
logger.info(
f"[REASON] clean-release preparation candidate={candidate_id} status={status}"
)
_append_event(
repository,
{
@@ -43,7 +47,9 @@ def audit_check_run(
candidate_id: Optional[str] = None,
actor: str = "system",
) -> None:
logger.info(f"[REFLECT] clean-release check_run={check_run_id} final_status={final_status}")
logger.info(
f"[REFLECT] clean-release check_run={check_run_id} final_status={final_status}"
)
_append_event(
repository,
{
@@ -67,7 +73,9 @@ def audit_violation(
candidate_id: Optional[str] = None,
actor: str = "system",
) -> None:
logger.info(f"[EXPLORE] clean-release violation run_id={run_id} stage={stage_name} code={code}")
logger.info(
f"[EXPLORE] clean-release violation run_id={run_id} stage={stage_name} code={code}"
)
_append_event(
repository,
{
@@ -91,7 +99,9 @@ def audit_report(
run_id: Optional[str] = None,
actor: str = "system",
) -> None:
logger.info(f"[EXPLORE] clean-release report_id={report_id} candidate={candidate_id}")
logger.info(
f"[EXPLORE] clean-release report_id={report_id} candidate={candidate_id}"
)
_append_event(
repository,
{
@@ -104,4 +114,6 @@ def audit_report(
"timestamp": datetime.now(timezone.utc).isoformat(),
},
)
# [/DEF:backend.src.services.clean_release.audit_service:Module]
# [/DEF:backend.src.services.clean_release.audit_service:Module]

94
backend/src/services/clean_release/compliance_execution_service.py
View File

@@ -1,23 +1,29 @@
# [DEF:compliance_execution_service:Module]
# [DEF:ComplianceExecutionService:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, compliance, execution, stages, immutable-evidence
# @PURPOSE: Create and execute compliance runs with trusted snapshots, deterministic stages, violations and immutable report persistence.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.policy_resolution_service
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.stages
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.report_builder
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @RELATION: [DEPENDS_ON] ->[PolicyResolutionService]
# @RELATION: [DEPENDS_ON] ->[ComplianceStages]
# @RELATION: [DEPENDS_ON] ->[ReportBuilder]
# @INVARIANT: A run binds to exactly one candidate/manifest/policy/registry snapshot set.
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Iterable, List, Optional
from typing import Any, Iterable, List, Optional, cast
from uuid import uuid4
from ...core.logger import belief_scope, logger
from ...models.clean_release import ComplianceReport, ComplianceRun, ComplianceStageRun, ComplianceViolation, DistributionManifest
from ...models.clean_release import (
ComplianceReport,
ComplianceRun,
ComplianceStageRun,
ComplianceViolation,
DistributionManifest,
)
from .audit_service import audit_check_run, audit_report, audit_violation
from .enums import ComplianceDecision, RunStatus
from .exceptions import ComplianceRunError, PolicyResolutionError
@@ -28,6 +34,9 @@ from .stages import build_default_stages, derive_final_status
from .stages.base import ComplianceStage, ComplianceStageContext, build_stage_run_record
belief_logger = cast(Any, logger)
# [DEF:ComplianceExecutionResult:Class]
# @PURPOSE: Return envelope for compliance execution with run/report and persisted stage artifacts.
@dataclass
@@ -36,6 +45,8 @@ class ComplianceExecutionResult:
report: Optional[ComplianceReport]
stage_runs: List[ComplianceStageRun]
violations: List[ComplianceViolation]
# [/DEF:ComplianceExecutionResult:Class]
@@ -45,6 +56,7 @@ class ComplianceExecutionResult:
# @POST: run state, stage records, violations and optional report are persisted consistently.
class ComplianceExecutionService:
TASK_PLUGIN_ID = "clean-release-compliance"
def __init__(
self,
*,
@@ -61,28 +73,36 @@ class ComplianceExecutionService:
# @PURPOSE: Resolve explicit manifest or fallback to latest candidate manifest.
# @PRE: candidate exists.
# @POST: Returns manifest snapshot or raises ComplianceRunError.
def _resolve_manifest(self, candidate_id: str, manifest_id: Optional[str]) -> DistributionManifest:
def _resolve_manifest(
self, candidate_id: str, manifest_id: Optional[str]
) -> DistributionManifest:
with belief_scope("ComplianceExecutionService._resolve_manifest"):
if manifest_id:
manifest = self.repository.get_manifest(manifest_id)
manifest_id_value = cast(Optional[str], manifest_id)
if manifest_id_value is not None and manifest_id_value != "":
manifest = self.repository.get_manifest(manifest_id_value)
if manifest is None:
raise ComplianceRunError(f"manifest '{manifest_id}' not found")
if manifest.candidate_id != candidate_id:
raise ComplianceRunError(
f"manifest '{manifest_id_value}' not found"
)
if str(getattr(manifest, "candidate_id")) != candidate_id:
raise ComplianceRunError("manifest does not belong to candidate")
return manifest
manifests = self.repository.get_manifests_by_candidate(candidate_id)
if not manifests:
raise ComplianceRunError(f"candidate '{candidate_id}' has no manifest")
return sorted(manifests, key=lambda item: item.manifest_version, reverse=True)[0]
return sorted(
manifests, key=lambda item: item.manifest_version, reverse=True
)[0]
# [/DEF:_resolve_manifest:Function]
# [DEF:_persist_stage_run:Function]
# @PURPOSE: Persist stage run if repository supports stage records.
# @POST: Stage run is persisted when adapter is available, otherwise no-op.
def _persist_stage_run(self, stage_run: ComplianceStageRun) -> None:
if hasattr(self.repository, "save_stage_run"):
self.repository.save_stage_run(stage_run)
self.repository.save_stage_run(stage_run)
# [/DEF:_persist_stage_run:Function]
# [DEF:_persist_violations:Function]
@@ -91,6 +111,7 @@ class ComplianceExecutionService:
def _persist_violations(self, violations: List[ComplianceViolation]) -> None:
for violation in violations:
self.repository.save_violation(violation)
# [/DEF:_persist_violations:Function]
# [DEF:execute_run:Function]
@@ -105,7 +126,9 @@ class ComplianceExecutionService:
manifest_id: Optional[str] = None,
) -> ComplianceExecutionResult:
with belief_scope("ComplianceExecutionService.execute_run"):
logger.reason(f"Starting compliance execution candidate_id={candidate_id}")
belief_logger.reason(
f"Starting compliance execution candidate_id={candidate_id}"
)
candidate = self.repository.get_candidate(candidate_id)
if candidate is None:
@@ -124,10 +147,10 @@ class ComplianceExecutionService:
run = ComplianceRun(
id=f"run-{uuid4()}",
candidate_id=candidate_id,
manifest_id=manifest.id,
manifest_digest=manifest.manifest_digest,
policy_snapshot_id=policy_snapshot.id,
registry_snapshot_id=registry_snapshot.id,
manifest_id=str(getattr(manifest, "id")),
manifest_digest=str(getattr(manifest, "manifest_digest")),
policy_snapshot_id=str(getattr(policy_snapshot, "id")),
registry_snapshot_id=str(getattr(registry_snapshot, "id")),
requested_by=requested_by,
requested_at=datetime.now(timezone.utc),
started_at=datetime.now(timezone.utc),
@@ -154,7 +177,7 @@ class ComplianceExecutionService:
finished = datetime.now(timezone.utc)
stage_run = build_stage_run_record(
run_id=run.id,
run_id=str(getattr(run, "id")),
stage_name=stage.stage_name,
result=result,
started_at=started,
@@ -167,23 +190,27 @@ class ComplianceExecutionService:
self._persist_violations(result.violations)
violations.extend(result.violations)
run.final_status = derive_final_status(stage_runs).value
run.status = RunStatus.SUCCEEDED.value
run.finished_at = datetime.now(timezone.utc)
setattr(run, "final_status", derive_final_status(stage_runs).value)
setattr(run, "status", RunStatus.SUCCEEDED.value)
setattr(run, "finished_at", datetime.now(timezone.utc))
self.repository.save_check_run(run)
report = self.report_builder.build_report_payload(run, violations)
report = self.report_builder.persist_report(report)
run.report_id = report.id
setattr(run, "report_id", str(getattr(report, "id")))
self.repository.save_check_run(run)
logger.reflect(f"[REFLECT] Compliance run completed run_id={run.id} final_status={run.final_status}")
belief_logger.reflect(
f"[REFLECT] Compliance run completed run_id={getattr(run, 'id')} final_status={getattr(run, 'final_status', None)}"
)
except Exception as exc: # noqa: BLE001
run.status = RunStatus.FAILED.value
run.final_status = ComplianceDecision.ERROR.value
run.failure_reason = str(exc)
run.finished_at = datetime.now(timezone.utc)
setattr(run, "status", RunStatus.FAILED.value)
setattr(run, "final_status", ComplianceDecision.ERROR.value)
setattr(run, "failure_reason", str(exc))
setattr(run, "finished_at", datetime.now(timezone.utc))
self.repository.save_check_run(run)
logger.explore(f"[EXPLORE] Compliance run failed run_id={run.id}: {exc}")
belief_logger.explore(
f"[EXPLORE] Compliance run failed run_id={getattr(run, 'id')}: {exc}"
)
return ComplianceExecutionResult(
run=run,
@@ -191,7 +218,10 @@ class ComplianceExecutionService:
stage_runs=stage_runs,
violations=violations,
)
# [/DEF:execute_run:Function]
# [/DEF:ComplianceExecutionService:Class]
# [/DEF:backend.src.services.clean_release.compliance_execution_service:Module]
# [/DEF:ComplianceExecutionService:Module]

68
backend/src/services/clean_release/compliance_orchestrator.py
View File

@@ -1,11 +1,11 @@
# [DEF:compliance_orchestrator:Module]
# [DEF:ComplianceOrchestrator:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, orchestrator, compliance-gate, stages
# @PURPOSE: Execute mandatory clean compliance stages and produce final COMPLIANT/BLOCKED/FAILED outcome.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.stages
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.report_builder
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: [DEPENDS_ON] ->[ComplianceStages]
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @INVARIANT: COMPLIANT is impossible when any mandatory stage fails.
# @TEST_CONTRACT: ComplianceCheckRun -> ComplianceCheckRun
# @TEST_FIXTURE: compliant_candidate -> file:backend/tests/fixtures/clean_release/fixtures_clean_release.json
@@ -51,7 +51,8 @@ class CleanComplianceOrchestrator:
def __init__(self, repository: CleanReleaseRepository):
with belief_scope("CleanComplianceOrchestrator.__init__"):
self.repository = repository
# [/DEF:CleanComplianceOrchestrator.__init__:Function]
# [/DEF:__init__:Function]
# [DEF:start_check_run:Function]
# @PURPOSE: Initiate a new compliance run session.
@@ -69,31 +70,51 @@ class CleanComplianceOrchestrator:
) -> ComplianceRun:
with belief_scope("start_check_run"):
actor = requested_by or legacy_kwargs.get("triggered_by") or "system"
execution_mode = str(legacy_kwargs.get("execution_mode") or "").strip().lower()
execution_mode = (
str(legacy_kwargs.get("execution_mode") or "").strip().lower()
)
manifest_id_value = manifest_id
if manifest_id_value and str(manifest_id_value).strip().lower() in {"tui", "api", "scheduler"}:
if manifest_id_value and str(manifest_id_value).strip().lower() in {
"tui",
"api",
"scheduler",
}:
logger.reason(
"Detected legacy positional execution_mode passed through manifest_id slot",
extra={"candidate_id": candidate_id, "execution_mode": manifest_id_value},
extra={
"candidate_id": candidate_id,
"execution_mode": manifest_id_value,
},
)
execution_mode = str(manifest_id_value).strip().lower()
manifest_id_value = None
manifest = self.repository.get_manifest(manifest_id_value) if manifest_id_value else None
manifest = (
self.repository.get_manifest(manifest_id_value)
if manifest_id_value
else None
)
policy = self.repository.get_policy(policy_id)
if manifest_id_value and manifest is None:
logger.explore(
"Manifest lookup missed during run start; rejecting explicit manifest contract",
extra={"candidate_id": candidate_id, "manifest_id": manifest_id_value},
extra={
"candidate_id": candidate_id,
"manifest_id": manifest_id_value,
},
)
raise ValueError("Manifest or Policy not found")
if policy is None:
logger.explore(
"Policy lookup missed during run start; using compatibility placeholder snapshot",
extra={"candidate_id": candidate_id, "policy_id": policy_id, "execution_mode": execution_mode or "unspecified"},
extra={
"candidate_id": candidate_id,
"policy_id": policy_id,
"execution_mode": execution_mode or "unspecified",
},
)
manifest_id_value = manifest_id_value or f"manifest-{candidate_id}"
@@ -118,9 +139,14 @@ class CleanComplianceOrchestrator:
)
logger.reflect(
"Initialized compliance run with compatibility-safe dependency placeholders",
extra={"run_id": check_run.id, "candidate_id": candidate_id, "policy_id": policy_id},
extra={
"run_id": check_run.id,
"candidate_id": candidate_id,
"policy_id": policy_id,
},
)
return self.repository.save_check_run(check_run)
# [/DEF:start_check_run:Function]
# [DEF:execute_stages:Function]
@@ -129,7 +155,11 @@ class CleanComplianceOrchestrator:
# @POST: Returns persisted ComplianceRun with status FAILED on missing dependencies, otherwise SUCCEEDED with final_status set.
# @SIDE_EFFECT: Reads candidate/policy/registry/manifest and persists updated check_run.
# @DATA_CONTRACT: Input -> (check_run:ComplianceRun, forced_results:Optional[List[ComplianceStageRun]]), Output -> ComplianceRun
def execute_stages(self, check_run: ComplianceRun, forced_results: Optional[List[ComplianceStageRun]] = None) -> ComplianceRun:
def execute_stages(
self,
check_run: ComplianceRun,
forced_results: Optional[List[ComplianceStageRun]] = None,
) -> ComplianceRun:
with belief_scope("execute_stages"):
if forced_results is not None:
for index, result in enumerate(forced_results, start=1):
@@ -170,12 +200,15 @@ class CleanComplianceOrchestrator:
summary = manifest.content_json.get("summary", {})
purity_ok = summary.get("prohibited_detected_count", 0) == 0
check_run.final_status = (
ComplianceDecision.PASSED.value if purity_ok else ComplianceDecision.BLOCKED.value
ComplianceDecision.PASSED.value
if purity_ok
else ComplianceDecision.BLOCKED.value
)
check_run.status = RunStatus.SUCCEEDED
check_run.finished_at = datetime.now(timezone.utc)
return self.repository.save_check_run(check_run)
# [/DEF:execute_stages:Function]
# [DEF:finalize_run:Function]
@@ -202,7 +235,10 @@ class CleanComplianceOrchestrator:
check_run.status = RunStatus.SUCCEEDED
check_run.finished_at = datetime.now(timezone.utc)
return self.repository.save_check_run(check_run)
# [/DEF:finalize_run:Function]
# [/DEF:CleanComplianceOrchestrator:Class]
@@ -229,5 +265,7 @@ def run_check_legacy(
)
run = orchestrator.execute_stages(run)
return orchestrator.finalize_run(run)
# [/DEF:run_check_legacy:Function]
# [/DEF:backend.src.services.clean_release.compliance_orchestrator:Module]
# [/DEF:ComplianceOrchestrator:Module]

12
backend/src/services/clean_release/demo_data_service.py
View File

@@ -1,9 +1,9 @@
# [DEF:demo_data_service:Module]
# [DEF:DemoDataService:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, demo-mode, namespace, isolation, repository
# @PURPOSE: Provide deterministic namespace helpers and isolated in-memory repository creation for demo and real modes.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @INVARIANT: Demo and real namespaces must never collide for generated physical identifiers.
from __future__ import annotations
@@ -20,6 +20,8 @@ def resolve_namespace(mode: str) -> str:
if normalized == "demo":
return "clean-release:demo"
return "clean-release:real"
# [/DEF:resolve_namespace:Function]
@@ -33,6 +35,8 @@ def build_namespaced_id(namespace: str, logical_id: str) -> str:
if not logical_id or not logical_id.strip():
raise ValueError("logical_id must be non-empty")
return f"{namespace}::{logical_id}"
# [/DEF:build_namespaced_id:Function]
@@ -45,6 +49,8 @@ def create_isolated_repository(mode: str) -> CleanReleaseRepository:
repository = CleanReleaseRepository()
setattr(repository, "namespace", namespace)
return repository
# [/DEF:create_isolated_repository:Function]
# [/DEF:backend.src.services.clean_release.demo_data_service:Module]
# [/DEF:DemoDataService:Module]

42
backend/src/services/clean_release/manifest_builder.py
View File

@@ -1,9 +1,9 @@
# [DEF:manifest_builder:Module]
# [DEF:ManifestBuilder:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, manifest, deterministic-hash, summary
# @PURPOSE: Build deterministic distribution manifest from classified artifact input.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @INVARIANT: Equal semantic artifact sets produce identical deterministic hash values.
from __future__ import annotations
@@ -21,7 +21,9 @@ from ...models.clean_release import (
)
def _stable_hash_payload(candidate_id: str, policy_id: str, items: List[ManifestItem]) -> str:
def _stable_hash_payload(
candidate_id: str, policy_id: str, items: List[ManifestItem]
) -> str:
serialized = [
{
"path": item.path,
@@ -30,14 +32,25 @@ def _stable_hash_payload(candidate_id: str, policy_id: str, items: List[Manifest
"reason": item.reason,
"checksum": item.checksum,
}
for item in sorted(items, key=lambda i: (i.path, i.category, i.classification.value, i.reason, i.checksum or ""))
for item in sorted(
items,
key=lambda i: (
i.path,
i.category,
i.classification.value,
i.reason,
i.checksum or "",
),
)
]
payload = {
"candidate_id": candidate_id,
"policy_id": policy_id,
"items": serialized,
}
digest = hashlib.sha256(json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest()
digest = hashlib.sha256(
json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest()
return digest
@@ -63,8 +76,17 @@ def build_distribution_manifest(
for a in artifacts
]
included_count = sum(1 for item in items if item.classification in {ClassificationType.REQUIRED_SYSTEM, ClassificationType.ALLOWED})
excluded_count = sum(1 for item in items if item.classification == ClassificationType.EXCLUDED_PROHIBITED)
included_count = sum(
1
for item in items
if item.classification
in {ClassificationType.REQUIRED_SYSTEM, ClassificationType.ALLOWED}
)
excluded_count = sum(
1
for item in items
if item.classification == ClassificationType.EXCLUDED_PROHIBITED
)
prohibited_detected_count = excluded_count
summary = ManifestSummary(
@@ -84,6 +106,8 @@ def build_distribution_manifest(
summary=summary,
deterministic_hash=deterministic_hash,
)
# [/DEF:build_distribution_manifest:Function]
@@ -105,5 +129,7 @@ def build_manifest(
generated_by=generated_by,
artifacts=artifacts,
)
# [/DEF:build_manifest:Function]
# [/DEF:backend.src.services.clean_release.manifest_builder:Module]
# [/DEF:ManifestBuilder:Module]

21
backend/src/services/clean_release/manifest_service.py
View File

@@ -1,11 +1,11 @@
# [DEF:manifest_service:Module]
# [DEF:ManifestService:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, manifest, versioning, immutability, lifecycle
# @PURPOSE: Build immutable distribution manifests with deterministic digest and version increment.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.manifest_builder
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @RELATION: [DEPENDS_ON] ->[ManifestBuilder]
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @PRE: Candidate exists and is PREPARED or MANIFEST_BUILT; artifacts are present.
# @POST: New immutable manifest is persisted with incremented version and deterministic digest.
# @INVARIANT: Existing manifests are never mutated.
@@ -39,8 +39,13 @@ def build_manifest_snapshot(
if candidate is None:
raise ValueError(f"candidate '{candidate_id}' not found")
if candidate.status not in {CandidateStatus.PREPARED.value, CandidateStatus.MANIFEST_BUILT.value}:
raise ValueError("candidate must be PREPARED or MANIFEST_BUILT to build manifest")
if candidate.status not in {
CandidateStatus.PREPARED.value,
CandidateStatus.MANIFEST_BUILT.value,
}:
raise ValueError(
"candidate must be PREPARED or MANIFEST_BUILT to build manifest"
)
artifacts = repository.get_artifacts_by_candidate(candidate_id)
if not artifacts:
@@ -83,6 +88,8 @@ def build_manifest_snapshot(
repository.save_candidate(candidate)
return manifest
# [/DEF:build_manifest_snapshot:Function]
# [/DEF:backend.src.services.clean_release.manifest_service:Module]
# [/DEF:ManifestService:Module]

76
backend/src/services/clean_release/policy_engine.py
View File

@@ -1,10 +1,10 @@
# [DEF:policy_engine:Module]
# [DEF:PolicyEngine:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, policy, classification, source-isolation
# @PURPOSE: Evaluate artifact/source policies for enterprise clean profile with deterministic outcomes.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release.CleanProfilePolicy
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release.ResourceSourceRegistry
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @RELATION: [DEPENDS_ON] ->[LoggerModule]
# @INVARIANT: Enterprise-clean policy always treats non-registry sources as violations.
from __future__ import annotations
@@ -54,7 +54,9 @@ class CleanPolicyEngine:
def validate_policy(self) -> PolicyValidationResult:
with belief_scope("clean_policy_engine.validate_policy"):
logger.reason("Validating enterprise-clean policy and internal registry consistency")
logger.reason(
"Validating enterprise-clean policy and internal registry consistency"
)
reasons: List[str] = []
registry_ref = (
@@ -68,38 +70,56 @@ class CleanPolicyEngine:
content = dict(getattr(self.policy, "content_json", None) or {})
if not content:
content = {
"profile": getattr(getattr(self.policy, "profile", None), "value", getattr(self.policy, "profile", "standard")),
"profile": getattr(
getattr(self.policy, "profile", None),
"value",
getattr(self.policy, "profile", "standard"),
),
"prohibited_artifact_categories": list(
getattr(self.policy, "prohibited_artifact_categories", []) or []
),
"required_system_categories": list(
getattr(self.policy, "required_system_categories", []) or []
),
"external_source_forbidden": getattr(self.policy, "external_source_forbidden", False),
"external_source_forbidden": getattr(
self.policy, "external_source_forbidden", False
),
}
profile = content.get("profile", "standard")
if profile == "enterprise-clean":
if not content.get("prohibited_artifact_categories"):
reasons.append("Enterprise policy requires prohibited artifact categories")
reasons.append(
"Enterprise policy requires prohibited artifact categories"
)
if not content.get("external_source_forbidden"):
reasons.append("Enterprise policy requires external_source_forbidden=true")
reasons.append(
"Enterprise policy requires external_source_forbidden=true"
)
registry_id = getattr(self.registry, "id", None) or getattr(self.registry, "registry_id", None)
registry_id = getattr(self.registry, "id", None) or getattr(
self.registry, "registry_id", None
)
if registry_id != registry_ref:
reasons.append("Policy registry ref does not match provided registry")
allowed_hosts = getattr(self.registry, "allowed_hosts", None)
if allowed_hosts is None:
entries = getattr(self.registry, "entries", []) or []
allowed_hosts = [entry.host for entry in entries if getattr(entry, "enabled", True)]
allowed_hosts = [
entry.host for entry in entries if getattr(entry, "enabled", True)
]
if not allowed_hosts:
reasons.append("Registry must contain allowed hosts")
logger.reflect(f"Policy validation completed. blocking_reasons={len(reasons)}")
return PolicyValidationResult(ok=len(reasons) == 0, blocking_reasons=reasons)
logger.reflect(
f"Policy validation completed. blocking_reasons={len(reasons)}"
)
return PolicyValidationResult(
ok=len(reasons) == 0, blocking_reasons=reasons
)
def classify_artifact(self, artifact: Dict) -> str:
category = (artifact.get("category") or "").strip()
@@ -118,10 +138,14 @@ class CleanPolicyEngine:
prohibited = content.get("prohibited_artifact_categories", [])
if category in required:
logger.reason(f"Artifact category '{category}' classified as required-system")
logger.reason(
f"Artifact category '{category}' classified as required-system"
)
return "required-system"
if category in prohibited:
logger.reason(f"Artifact category '{category}' classified as excluded-prohibited")
logger.reason(
f"Artifact category '{category}' classified as excluded-prohibited"
)
return "excluded-prohibited"
logger.reflect(f"Artifact category '{category}' classified as allowed")
return "allowed"
@@ -129,7 +153,9 @@ class CleanPolicyEngine:
def validate_resource_source(self, endpoint: str) -> SourceValidationResult:
with belief_scope("clean_policy_engine.validate_resource_source"):
if not endpoint:
logger.explore("Empty endpoint detected; treating as blocking external-source violation")
logger.explore(
"Empty endpoint detected; treating as blocking external-source violation"
)
return SourceValidationResult(
ok=False,
violation={
@@ -143,12 +169,16 @@ class CleanPolicyEngine:
allowed_hosts = getattr(self.registry, "allowed_hosts", None)
if allowed_hosts is None:
entries = getattr(self.registry, "entries", []) or []
allowed_hosts = [entry.host for entry in entries if getattr(entry, "enabled", True)]
allowed_hosts = [
entry.host for entry in entries if getattr(entry, "enabled", True)
]
allowed_hosts = set(allowed_hosts or [])
normalized = endpoint.strip().lower()
if normalized in allowed_hosts:
logger.reason(f"Endpoint '{normalized}' is present in internal allowlist")
logger.reason(
f"Endpoint '{normalized}' is present in internal allowlist"
)
return SourceValidationResult(ok=True, violation=None)
logger.explore(f"Endpoint '{endpoint}' is outside internal allowlist")
@@ -162,9 +192,13 @@ class CleanPolicyEngine:
},
)
def evaluate_candidate(self, artifacts: Iterable[Dict], sources: Iterable[str]) -> Tuple[List[Dict], List[Dict]]:
def evaluate_candidate(
self, artifacts: Iterable[Dict], sources: Iterable[str]
) -> Tuple[List[Dict], List[Dict]]:
with belief_scope("clean_policy_engine.evaluate_candidate"):
logger.reason("Evaluating candidate artifacts and resource sources against enterprise policy")
logger.reason(
"Evaluating candidate artifacts and resource sources against enterprise policy"
)
classified: List[Dict] = []
violations: List[Dict] = []
@@ -192,5 +226,7 @@ class CleanPolicyEngine:
f"Candidate evaluation finished. artifacts={len(classified)} violations={len(violations)}"
)
return classified, violations
# [/DEF:CleanPolicyEngine:Class]
# [/DEF:backend.src.services.clean_release.policy_engine:Module]
# [/DEF:PolicyEngine:Module]

36
backend/src/services/clean_release/policy_resolution_service.py
View File

@@ -1,11 +1,11 @@
# [DEF:policy_resolution_service:Module]
# [DEF:PolicyResolutionService:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, policy, registry, trusted-resolution, immutable-snapshots
# @PURPOSE: Resolve trusted policy and registry snapshots from ConfigManager without runtime overrides.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.core.config_manager
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.exceptions
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @RELATION: [DEPENDS_ON] ->[clean_release_exceptions]
# @INVARIANT: Trusted snapshot resolution is based only on ConfigManager active identifiers.
from __future__ import annotations
@@ -30,10 +30,14 @@ def resolve_trusted_policy_snapshots(
registry_id_override: Optional[str] = None,
) -> Tuple[CleanPolicySnapshot, SourceRegistrySnapshot]:
if policy_id_override is not None or registry_id_override is not None:
raise PolicyResolutionError("override attempt is forbidden for trusted policy resolution")
raise PolicyResolutionError(
"override attempt is forbidden for trusted policy resolution"
)
config = config_manager.get_config()
clean_release_settings = getattr(getattr(config, "settings", None), "clean_release", None)
clean_release_settings = getattr(
getattr(config, "settings", None), "clean_release", None
)
if clean_release_settings is None:
raise PolicyResolutionError("clean_release settings are missing")
@@ -41,17 +45,25 @@ def resolve_trusted_policy_snapshots(
registry_id = getattr(clean_release_settings, "active_registry_id", None)
if not policy_id:
raise PolicyResolutionError("missing trusted profile: active_policy_id is not configured")
raise PolicyResolutionError(
"missing trusted profile: active_policy_id is not configured"
)
if not registry_id:
raise PolicyResolutionError("missing trusted registry: active_registry_id is not configured")
raise PolicyResolutionError(
"missing trusted registry: active_registry_id is not configured"
)
policy_snapshot = repository.get_policy(policy_id)
if policy_snapshot is None:
raise PolicyResolutionError(f"trusted policy snapshot '{policy_id}' was not found")
raise PolicyResolutionError(
f"trusted policy snapshot '{policy_id}' was not found"
)
registry_snapshot = repository.get_registry(registry_id)
if registry_snapshot is None:
raise PolicyResolutionError(f"trusted registry snapshot '{registry_id}' was not found")
raise PolicyResolutionError(
f"trusted registry snapshot '{registry_id}' was not found"
)
if not bool(getattr(policy_snapshot, "immutable", False)):
raise PolicyResolutionError("policy snapshot must be immutable")
@@ -59,6 +71,8 @@ def resolve_trusted_policy_snapshots(
raise PolicyResolutionError("registry snapshot must be immutable")
return policy_snapshot, registry_snapshot
# [/DEF:resolve_trusted_policy_snapshots:Function]
# [/DEF:backend.src.services.clean_release.policy_resolution_service:Module]
# [/DEF:PolicyResolutionService:Module]

31
backend/src/services/clean_release/preparation_service.py
View File

@@ -1,11 +1,11 @@
# [DEF:preparation_service:Module]
# [DEF:PreparationService:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, preparation, manifest, policy-evaluation
# @PURPOSE: Prepare release candidate by policy evaluation and deterministic manifest creation.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.policy_engine
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.manifest_builder
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: [DEPENDS_ON] ->[PolicyEngine]
# @RELATION: [DEPENDS_ON] ->[ManifestBuilder]
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @INVARIANT: Candidate preparation always persists manifest and candidate status deterministically.
from __future__ import annotations
@@ -35,9 +35,8 @@ def prepare_candidate(
if policy is None:
raise ValueError("Active clean policy not found")
registry_ref = (
getattr(policy, "registry_snapshot_id", None)
or getattr(policy, "internal_source_registry_ref", None)
registry_ref = getattr(policy, "registry_snapshot_id", None) or getattr(
policy, "internal_source_registry_ref", None
)
registry = repository.get_registry(registry_ref) if registry_ref else None
if registry is None:
@@ -48,7 +47,9 @@ def prepare_candidate(
if not validation.ok:
raise ValueError(f"Invalid policy: {validation.blocking_reasons}")
classified, violations = engine.evaluate_candidate(artifacts=artifacts, sources=sources)
classified, violations = engine.evaluate_candidate(
artifacts=artifacts, sources=sources
)
manifest = build_distribution_manifest(
manifest_id=f"manifest-{candidate_id}",
@@ -65,14 +66,20 @@ def prepare_candidate(
repository.save_candidate(candidate)
response_status = ReleaseCandidateStatus.BLOCKED.value
else:
if current_status in {CandidateStatus.DRAFT, CandidateStatus.DRAFT.value, "DRAFT"}:
if current_status in {
CandidateStatus.DRAFT,
CandidateStatus.DRAFT.value,
"DRAFT",
}:
candidate.transition_to(CandidateStatus.PREPARED)
else:
candidate.status = ReleaseCandidateStatus.PREPARED.value
repository.save_candidate(candidate)
response_status = ReleaseCandidateStatus.PREPARED.value
manifest_id_value = getattr(manifest, "manifest_id", None) or getattr(manifest, "id", "")
manifest_id_value = getattr(manifest, "manifest_id", None) or getattr(
manifest, "id", ""
)
return {
"candidate_id": candidate_id,
"status": response_status,
@@ -100,5 +107,7 @@ def prepare_candidate_legacy(
sources=sources,
operator_id=operator_id,
)
# [/DEF:prepare_candidate_legacy:Function]
# [/DEF:backend.src.services.clean_release.preparation_service:Module]
# [/DEF:PreparationService:Module]

78
backend/src/services/clean_release/publication_service.py
View File

@@ -1,12 +1,12 @@
# [DEF:publication_service:Module]
# [DEF:PublicationService:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, publication, revoke, gate, lifecycle
# @PURPOSE: Enforce publication and revocation gates with append-only publication records.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.approval_service
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.audit_service
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @RELATION: [DEPENDS_ON] ->[ApprovalService]
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @RELATION: [DEPENDS_ON] ->[AuditService]
# @INVARIANT: Publication records are append-only snapshots; revoke mutates only publication status for targeted record.
from __future__ import annotations
@@ -27,12 +27,16 @@ from .repository import CleanReleaseRepository
# @PURPOSE: Provide in-memory append-only publication storage.
# @PRE: repository is initialized.
# @POST: Returns publication list attached to repository.
def _get_or_init_publications_store(repository: CleanReleaseRepository) -> List[PublicationRecord]:
def _get_or_init_publications_store(
repository: CleanReleaseRepository,
) -> List[PublicationRecord]:
publications = getattr(repository, "publication_records", None)
if publications is None:
publications = []
setattr(repository, "publication_records", publications)
return publications
# [/DEF:_get_or_init_publications_store:Function]
@@ -44,10 +48,20 @@ def _latest_publication_for_candidate(
repository: CleanReleaseRepository,
candidate_id: str,
) -> PublicationRecord | None:
records = [item for item in _get_or_init_publications_store(repository) if item.candidate_id == candidate_id]
records = [
item
for item in _get_or_init_publications_store(repository)
if item.candidate_id == candidate_id
]
if not records:
return None
return sorted(records, key=lambda item: item.published_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True)[0]
return sorted(
records,
key=lambda item: item.published_at or datetime.min.replace(tzinfo=timezone.utc),
reverse=True,
)[0]
# [/DEF:_latest_publication_for_candidate:Function]
@@ -55,12 +69,20 @@ def _latest_publication_for_candidate(
# @PURPOSE: Resolve latest approval decision from repository decision store.
# @PRE: candidate_id is non-empty.
# @POST: Returns latest decision object or None.
def _latest_approval_for_candidate(repository: CleanReleaseRepository, candidate_id: str):
def _latest_approval_for_candidate(
repository: CleanReleaseRepository, candidate_id: str
):
decisions = getattr(repository, "approval_decisions", [])
scoped = [item for item in decisions if item.candidate_id == candidate_id]
if not scoped:
return None
return sorted(scoped, key=lambda item: item.decided_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True)[0]
return sorted(
scoped,
key=lambda item: item.decided_at or datetime.min.replace(tzinfo=timezone.utc),
reverse=True,
)[0]
# [/DEF:_latest_approval_for_candidate:Function]
@@ -78,7 +100,9 @@ def publish_candidate(
publication_ref: str | None = None,
) -> PublicationRecord:
with belief_scope("publication_service.publish_candidate"):
logger.reason(f"[REASON] Evaluating publish gate candidate_id={candidate_id} report_id={report_id}")
logger.reason(
f"[REASON] Evaluating publish gate candidate_id={candidate_id} report_id={report_id}"
)
if not published_by or not published_by.strip():
raise PublicationGateError("published_by must be non-empty")
@@ -96,11 +120,17 @@ def publish_candidate(
raise PublicationGateError("report belongs to another candidate")
latest_approval = _latest_approval_for_candidate(repository, candidate_id)
if latest_approval is None or latest_approval.decision != ApprovalDecisionType.APPROVED.value:
if (
latest_approval is None
or latest_approval.decision != ApprovalDecisionType.APPROVED.value
):
raise PublicationGateError("publish requires APPROVED decision")
latest_publication = _latest_publication_for_candidate(repository, candidate_id)
if latest_publication is not None and latest_publication.status == PublicationStatus.ACTIVE.value:
if (
latest_publication is not None
and latest_publication.status == PublicationStatus.ACTIVE.value
):
raise PublicationGateError("candidate already has active publication")
if candidate.status == CandidateStatus.APPROVED.value:
@@ -108,7 +138,9 @@ def publish_candidate(
candidate.transition_to(CandidateStatus.PUBLISHED)
repository.save_candidate(candidate)
except Exception as exc: # noqa: BLE001
logger.explore(f"[EXPLORE] Candidate transition to PUBLISHED failed candidate_id={candidate_id}: {exc}")
logger.explore(
f"[EXPLORE] Candidate transition to PUBLISHED failed candidate_id={candidate_id}: {exc}"
)
raise PublicationGateError(str(exc)) from exc
record = PublicationRecord(
@@ -122,9 +154,15 @@ def publish_candidate(
status=PublicationStatus.ACTIVE.value,
)
_get_or_init_publications_store(repository).append(record)
audit_preparation(candidate_id, "PUBLISHED", repository=repository, actor=published_by)
logger.reflect(f"[REFLECT] Publication persisted candidate_id={candidate_id} publication_id={record.id}")
audit_preparation(
candidate_id, "PUBLISHED", repository=repository, actor=published_by
)
logger.reflect(
f"[REFLECT] Publication persisted candidate_id={candidate_id} publication_id={record.id}"
)
return record
# [/DEF:publish_candidate:Function]
@@ -140,7 +178,9 @@ def revoke_publication(
comment: str | None = None,
) -> PublicationRecord:
with belief_scope("publication_service.revoke_publication"):
logger.reason(f"[REASON] Evaluating revoke gate publication_id={publication_id}")
logger.reason(
f"[REASON] Evaluating revoke gate publication_id={publication_id}"
)
if not revoked_by or not revoked_by.strip():
raise PublicationGateError("revoked_by must be non-empty")
@@ -168,6 +208,8 @@ def revoke_publication(
)
logger.reflect(f"[REFLECT] Publication revoked publication_id={publication_id}")
return record
# [/DEF:revoke_publication:Function]
# [/DEF:backend.src.services.clean_release.publication_service:Module]
# [/DEF:backend.src.services.clean_release.publication_service:Module]

19
backend/src/services/clean_release/report_builder.py
View File

@@ -1,10 +1,10 @@
# [DEF:report_builder:Module]
# [DEF:ReportBuilder:Module]
# @COMPLEXITY: 5
# @SEMANTICS: clean-release, report, audit, counters, violations
# @PURPOSE: Build and persist compliance reports with consistent counter invariants.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @RELATION: [DEPENDS_ON] ->[RepositoryRelations]
# @INVARIANT: blocking_violations_count never exceeds violations_count.
# @TEST_CONTRACT: ComplianceCheckRun,List[ComplianceViolation] -> ComplianceReport
# @TEST_FIXTURE: blocked_with_two_violations -> file:backend/tests/fixtures/clean_release/fixtures_clean_release.json
@@ -28,7 +28,9 @@ class ComplianceReportBuilder:
def __init__(self, repository: CleanReleaseRepository):
self.repository = repository
def build_report_payload(self, check_run: ComplianceRun, violations: List[ComplianceViolation]) -> ComplianceReport:
def build_report_payload(
self, check_run: ComplianceRun, violations: List[ComplianceViolation]
) -> ComplianceReport:
if check_run.status == RunStatus.RUNNING:
raise ValueError("Cannot build report for non-terminal run")
@@ -40,7 +42,10 @@ class ComplianceReportBuilder:
or bool(getattr(v, "evidence_json", {}).get("blocked_release", False))
)
if check_run.final_status == ComplianceDecision.BLOCKED and blocking_violations_count <= 0:
if (
check_run.final_status == ComplianceDecision.BLOCKED
and blocking_violations_count <= 0
):
raise ValueError("Blocked run requires at least one blocking violation")
summary = (
@@ -65,4 +70,6 @@ class ComplianceReportBuilder:
def persist_report(self, report: ComplianceReport) -> ComplianceReport:
return self.repository.save_report(report)
# [/DEF:backend.src.services.clean_release.report_builder:Module]
# [/DEF:ReportBuilder:Module]

96
backend/src/services/clean_release/repository.py
View File

@@ -1,15 +1,15 @@
# [DEF:repository:Module]
# [DEF:RepositoryRelations:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, repository, persistence, in-memory
# @PURPOSE: Provide repository adapter for clean release entities with deterministic access methods.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @INVARIANT: Repository operations are side-effect free outside explicit save/update calls.
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, cast
from ...models.clean_release import (
CleanPolicySnapshot,
@@ -38,15 +38,28 @@ class CleanReleaseRepository:
violations: Dict[str, ComplianceViolation] = field(default_factory=dict)
audit_events: List[Dict[str, Any]] = field(default_factory=list)
def _entity_id(self, entity: Any) -> str:
return cast(str, str(getattr(cast(Any, entity), "id")))
def _candidate_ref(self, entity: Any) -> str:
return cast(str, str(getattr(cast(Any, entity), "candidate_id")))
def _run_ref(self, entity: Any) -> str:
return cast(str, str(getattr(cast(Any, entity), "run_id")))
def save_candidate(self, candidate: ReleaseCandidate) -> ReleaseCandidate:
self.candidates[candidate.id] = candidate
cast(Dict[str, ReleaseCandidate], self.candidates)[
self._entity_id(candidate)
] = candidate
return candidate
def get_candidate(self, candidate_id: str) -> Optional[ReleaseCandidate]:
return self.candidates.get(candidate_id)
def save_policy(self, policy: CleanPolicySnapshot) -> CleanPolicySnapshot:
self.policies[policy.id] = policy
cast(Dict[str, CleanPolicySnapshot], self.policies)[self._entity_id(policy)] = (
policy
)
return policy
def get_policy(self, policy_id: str) -> Optional[CleanPolicySnapshot]:
@@ -58,36 +71,57 @@ class CleanReleaseRepository:
return next(iter(self.policies.values()), None)
def save_registry(self, registry: SourceRegistrySnapshot) -> SourceRegistrySnapshot:
self.registries[registry.id] = registry
cast(Dict[str, SourceRegistrySnapshot], self.registries)[
self._entity_id(registry)
] = registry
return registry
def get_registry(self, registry_id: str) -> Optional[SourceRegistrySnapshot]:
return self.registries.get(registry_id)
def save_artifact(self, artifact) -> object:
self.artifacts[artifact.id] = artifact
cast(Dict[str, object], self.artifacts)[self._entity_id(artifact)] = artifact
return artifact
def get_artifacts_by_candidate(self, candidate_id: str) -> List[object]:
return [a for a in self.artifacts.values() if a.candidate_id == candidate_id]
artifacts = cast(List[Any], list(self.artifacts.values()))
return [
artifact
for artifact in artifacts
if self._candidate_ref(artifact) == candidate_id
]
def save_manifest(self, manifest: DistributionManifest) -> DistributionManifest:
self.manifests[manifest.id] = manifest
cast(Dict[str, DistributionManifest], self.manifests)[
self._entity_id(manifest)
] = manifest
return manifest
def get_manifest(self, manifest_id: str) -> Optional[DistributionManifest]:
return self.manifests.get(manifest_id)
def save_distribution_manifest(self, manifest: DistributionManifest) -> DistributionManifest:
def save_distribution_manifest(
self, manifest: DistributionManifest
) -> DistributionManifest:
return self.save_manifest(manifest)
def get_distribution_manifest(self, manifest_id: str) -> Optional[DistributionManifest]:
def get_distribution_manifest(
self, manifest_id: str
) -> Optional[DistributionManifest]:
return self.get_manifest(manifest_id)
def save_check_run(self, check_run: ComplianceRun) -> ComplianceRun:
self.check_runs[check_run.id] = check_run
cast(Dict[str, ComplianceRun], self.check_runs)[self._entity_id(check_run)] = (
check_run
)
return check_run
def save_stage_run(self, stage_run: ComplianceStageRun) -> ComplianceStageRun:
cast(Dict[str, ComplianceStageRun], self.stage_runs)[
self._entity_id(stage_run)
] = stage_run
return stage_run
def get_check_run(self, check_run_id: str) -> Optional[ComplianceRun]:
return self.check_runs.get(check_run_id)
@@ -98,27 +132,49 @@ class CleanReleaseRepository:
return self.get_check_run(run_id)
def save_report(self, report: ComplianceReport) -> ComplianceReport:
existing = self.reports.get(report.id)
report_id = self._entity_id(report)
store = cast(Dict[str, ComplianceReport], self.reports)
existing = store.get(report_id)
if existing is not None:
raise ValueError(f"immutable report snapshot already exists for id={report.id}")
self.reports[report.id] = report
raise ValueError(
f"immutable report snapshot already exists for id={report_id}"
)
store[report_id] = report
return report
def get_report(self, report_id: str) -> Optional[ComplianceReport]:
return self.reports.get(report_id)
def save_violation(self, violation: ComplianceViolation) -> ComplianceViolation:
self.violations[violation.id] = violation
cast(Dict[str, ComplianceViolation], self.violations)[
self._entity_id(violation)
] = violation
return violation
def get_violations_by_run(self, run_id: str) -> List[ComplianceViolation]:
return [v for v in self.violations.values() if v.run_id == run_id]
violations = cast(List[ComplianceViolation], list(self.violations.values()))
return [
violation for violation in violations if self._run_ref(violation) == run_id
]
def get_manifests_by_candidate(
self, candidate_id: str
) -> List[DistributionManifest]:
manifests = cast(List[DistributionManifest], list(self.manifests.values()))
return [
manifest
for manifest in manifests
if self._candidate_ref(manifest) == candidate_id
]
def append_audit_event(self, payload: Dict[str, Any]) -> None:
self.audit_events.append(payload)
def get_manifests_by_candidate(self, candidate_id: str) -> List[DistributionManifest]:
return [m for m in self.manifests.values() if m.candidate_id == candidate_id]
def clear_history(self) -> None:
self.check_runs.clear()
self.reports.clear()
self.violations.clear()
# [/DEF:CleanReleaseRepository:Class]
# [/DEF:backend.src.services.clean_release.repository:Module]
# [/DEF:RepositoryRelations:Module]

16
backend/src/services/clean_release/source_isolation.py
View File

@@ -1,9 +1,9 @@
# [DEF:source_isolation:Module]
# [DEF:SourceIsolation:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, source-isolation, internal-only, validation
# @PURPOSE: Validate that all resource endpoints belong to the approved internal source registry.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release.ResourceSourceRegistry
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @INVARIANT: Any endpoint outside enabled registry entries is treated as external-source violation.
from __future__ import annotations
@@ -13,8 +13,12 @@ from typing import Dict, Iterable, List
from ...models.clean_release import ResourceSourceRegistry
def validate_internal_sources(registry: ResourceSourceRegistry, endpoints: Iterable[str]) -> Dict:
allowed_hosts = {entry.host.strip().lower() for entry in registry.entries if entry.enabled}
def validate_internal_sources(
registry: ResourceSourceRegistry, endpoints: Iterable[str]
) -> Dict:
allowed_hosts = {
entry.host.strip().lower() for entry in registry.entries if entry.enabled
}
violations: List[Dict] = []
for endpoint in endpoints:
@@ -30,4 +34,6 @@ def validate_internal_sources(registry: ResourceSourceRegistry, endpoints: Itera
)
return {"ok": len(violations) == 0, "violations": violations}
# [/DEF:backend.src.services.clean_release.source_isolation:Module]
# [/DEF:SourceIsolation:Module]

27
backend/src/services/clean_release/stages/__init__.py
View File

@@ -1,9 +1,10 @@
# [DEF:stages:Module]
# [DEF:ComplianceStages:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, compliance, stages, state-machine
# @PURPOSE: Define compliance stage order and helper functions for deterministic run-state evaluation.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @RELATION: [DEPENDS_ON] ->[ComplianceStageBase]
# @INVARIANT: Stage order remains deterministic for all compliance runs.
from __future__ import annotations
@@ -42,6 +43,8 @@ def build_default_stages() -> List[ComplianceStage]:
NoExternalEndpointsStage(),
ManifestConsistencyStage(),
]
# [/DEF:build_default_stages:Function]
@@ -55,7 +58,9 @@ def stage_result_map(
normalized: Dict[ComplianceStageName, CheckStageStatus] = {}
for result in stage_results:
if isinstance(result, CheckStageResult):
normalized[ComplianceStageName(result.stage.value)] = CheckStageStatus(result.status.value)
normalized[ComplianceStageName(result.stage.value)] = CheckStageStatus(
result.status.value
)
continue
stage_name = getattr(result, "stage_name", None)
@@ -77,6 +82,8 @@ def stage_result_map(
elif status:
normalized[normalized_stage] = CheckStageStatus(str(status))
return normalized
# [/DEF:stage_result_map:Function]
@@ -84,8 +91,12 @@ def stage_result_map(
# @PURPOSE: Identify mandatory stages that are absent from run results.
# @PRE: stage_status_map contains zero or more known stage statuses.
# @POST: Returns ordered list of missing mandatory stages.
def missing_mandatory_stages(stage_status_map: Dict[ComplianceStageName, CheckStageStatus]) -> List[ComplianceStageName]:
def missing_mandatory_stages(
stage_status_map: Dict[ComplianceStageName, CheckStageStatus],
) -> List[ComplianceStageName]:
return [stage for stage in MANDATORY_STAGE_ORDER if stage not in stage_status_map]
# [/DEF:missing_mandatory_stages:Function]
@@ -93,7 +104,9 @@ def missing_mandatory_stages(stage_status_map: Dict[ComplianceStageName, CheckSt
# @PURPOSE: Derive final run status from stage results with deterministic blocking behavior.
# @PRE: Stage statuses correspond to compliance checks.
# @POST: Returns one of PASSED/BLOCKED/ERROR according to mandatory stage outcomes.
def derive_final_status(stage_results: Iterable[ComplianceStageRun | CheckStageResult]) -> CheckFinalStatus:
def derive_final_status(
stage_results: Iterable[ComplianceStageRun | CheckStageResult],
) -> CheckFinalStatus:
status_map = stage_result_map(stage_results)
missing = missing_mandatory_stages(status_map)
if missing:
@@ -107,5 +120,7 @@ def derive_final_status(stage_results: Iterable[ComplianceStageRun | CheckStageR
return CheckFinalStatus.BLOCKED
return CheckFinalStatus.COMPLIANT
# [/DEF:derive_final_status:Function]
# [/DEF:backend.src.services.clean_release.stages:Module]
# [/DEF:ComplianceStages:Module]

25
backend/src/services/clean_release/stages/base.py
View File

@@ -1,10 +1,10 @@
# [DEF:base:Module]
# [DEF:ComplianceStageBase:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, compliance, stages, contracts, base
# @PURPOSE: Define shared contracts and helpers for pluggable clean-release compliance stages.
# @LAYER: Domain
# @RELATION: CALLED_BY -> backend.src.services.clean_release.compliance_execution_service
# @RELATION: DEPENDS_ON -> backend.src.models.clean_release
# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels]
# @RELATION: [DEPENDS_ON] ->[LoggerModule]
# @INVARIANT: Stage execution is deterministic for equal input context.
from __future__ import annotations
@@ -37,6 +37,8 @@ class ComplianceStageContext:
manifest: DistributionManifest
policy: CleanPolicySnapshot
registry: SourceRegistrySnapshot
# [/DEF:ComplianceStageContext:Class]
@@ -47,6 +49,8 @@ class StageExecutionResult:
decision: ComplianceDecision
details_json: Dict[str, Any] = field(default_factory=dict)
violations: List[ComplianceViolation] = field(default_factory=list)
# [/DEF:StageExecutionResult:Class]
@@ -55,8 +59,9 @@ class StageExecutionResult:
class ComplianceStage(Protocol):
stage_name: ComplianceStageName
def execute(self, context: ComplianceStageContext) -> StageExecutionResult:
...
def execute(self, context: ComplianceStageContext) -> StageExecutionResult: ...
# [/DEF:ComplianceStage:Class]
@@ -78,12 +83,16 @@ def build_stage_run_record(
id=f"stg-{uuid4()}",
run_id=run_id,
stage_name=stage_name.value,
status="SUCCEEDED" if result.decision != ComplianceDecision.ERROR else "FAILED",
status="SUCCEEDED"
if result.decision != ComplianceDecision.ERROR
else "FAILED",
started_at=started_at or now,
finished_at=finished_at or now,
decision=result.decision.value,
details_json=result.details_json,
)
# [/DEF:build_stage_run_record:Function]
@@ -118,6 +127,8 @@ def build_violation(
"blocked_release": blocked_release,
},
)
# [/DEF:build_violation:Function]
# [/DEF:backend.src.services.clean_release.stages.base:Module]
# [/DEF:ComplianceStageBase:Module]

8
backend/src/services/clean_release/stages/data_purity.py
View File

@@ -3,8 +3,8 @@
# @SEMANTICS: clean-release, compliance-stage, data-purity
# @PURPOSE: Evaluate manifest purity counters and emit blocking violations for prohibited artifacts.
# @LAYER: Domain
# @RELATION: IMPLEMENTS -> backend.src.services.clean_release.stages.base.ComplianceStage
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.stages.base
# @RELATION: [IMPLEMENTS] ->[ComplianceStage]
# @RELATION: [DEPENDS_ON] ->[ComplianceStageBase]
# @INVARIANT: prohibited_detected_count > 0 always yields BLOCKED stage decision.
from __future__ import annotations
@@ -61,6 +61,8 @@ class DataPurityStage:
},
violations=[violation],
)
# [/DEF:DataPurityStage:Class]
# [/DEF:backend.src.services.clean_release.stages.data_purity:Module]
# [/DEF:data_purity:Module]

23
backend/src/services/clean_release/stages/internal_sources_only.py
View File

@@ -3,8 +3,8 @@
# @SEMANTICS: clean-release, compliance-stage, source-isolation, registry
# @PURPOSE: Verify manifest-declared sources belong to trusted internal registry allowlist.
# @LAYER: Domain
# @RELATION: IMPLEMENTS -> backend.src.services.clean_release.stages.base.ComplianceStage
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.stages.base
# @RELATION: [IMPLEMENTS] ->[ComplianceStage]
# @RELATION: [DEPENDS_ON] ->[ComplianceStageBase]
# @INVARIANT: Any source host outside allowed_hosts yields BLOCKED decision with at least one violation.
from __future__ import annotations
@@ -23,7 +23,10 @@ class InternalSourcesOnlyStage:
def execute(self, context: ComplianceStageContext) -> StageExecutionResult:
with belief_scope("InternalSourcesOnlyStage.execute"):
allowed_hosts = {str(host).strip().lower() for host in (context.registry.allowed_hosts or [])}
allowed_hosts = {
str(host).strip().lower()
for host in (context.registry.allowed_hosts or [])
}
sources = context.manifest.content_json.get("sources", [])
violations = []
@@ -32,7 +35,11 @@ class InternalSourcesOnlyStage:
)
for source in sources:
host = str(source.get("host", "")).strip().lower() if isinstance(source, dict) else ""
host = (
str(source.get("host", "")).strip().lower()
if isinstance(source, dict)
else ""
)
if not host or host in allowed_hosts:
continue
@@ -42,7 +49,9 @@ class InternalSourcesOnlyStage:
stage_name=self.stage_name,
code="SOURCE_HOST_NOT_ALLOWED",
message=f"Source host '{host}' is not in trusted internal registry",
artifact_path=str(source.get("path", "")) if isinstance(source, dict) else None,
artifact_path=str(source.get("path", ""))
if isinstance(source, dict)
else None,
severity=ViolationSeverity.CRITICAL,
evidence_json={
"host": host,
@@ -71,6 +80,8 @@ class InternalSourcesOnlyStage:
},
violations=[],
)
# [/DEF:InternalSourcesOnlyStage:Class]
# [/DEF:backend.src.services.clean_release.stages.internal_sources_only:Module]
# [/DEF:internal_sources_only:Module]

8
backend/src/services/clean_release/stages/manifest_consistency.py
View File

@@ -3,8 +3,8 @@
# @SEMANTICS: clean-release, compliance-stage, manifest, consistency, digest
# @PURPOSE: Ensure run is bound to the exact manifest snapshot and digest used at run creation time.
# @LAYER: Domain
# @RELATION: IMPLEMENTS -> backend.src.services.clean_release.stages.base.ComplianceStage
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.stages.base
# @RELATION: [IMPLEMENTS] ->[ComplianceStage]
# @RELATION: [DEPENDS_ON] ->[ComplianceStageBase]
# @INVARIANT: Digest mismatch between run and manifest yields ERROR with blocking violation evidence.
from __future__ import annotations
@@ -65,6 +65,8 @@ class ManifestConsistencyStage:
},
violations=[violation],
)
# [/DEF:ManifestConsistencyStage:Class]
# [/DEF:backend.src.services.clean_release.stages.manifest_consistency:Module]
# [/DEF:manifest_consistency:Module]

23
backend/src/services/clean_release/stages/no_external_endpoints.py
View File

@@ -3,8 +3,8 @@
# @SEMANTICS: clean-release, compliance-stage, endpoints, network
# @PURPOSE: Block manifest payloads that expose external endpoints outside trusted schemes and hosts.
# @LAYER: Domain
# @RELATION: IMPLEMENTS -> backend.src.services.clean_release.stages.base.ComplianceStage
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.stages.base
# @RELATION: [IMPLEMENTS] ->[ComplianceStage]
# @RELATION: [DEPENDS_ON] ->[ComplianceStageBase]
# @INVARIANT: Endpoint outside allowed scheme/host always yields BLOCKED stage decision.
from __future__ import annotations
@@ -26,8 +26,14 @@ class NoExternalEndpointsStage:
def execute(self, context: ComplianceStageContext) -> StageExecutionResult:
with belief_scope("NoExternalEndpointsStage.execute"):
endpoints = context.manifest.content_json.get("endpoints", [])
allowed_hosts = {str(host).strip().lower() for host in (context.registry.allowed_hosts or [])}
allowed_schemes = {str(scheme).strip().lower() for scheme in (context.registry.allowed_schemes or [])}
allowed_hosts = {
str(host).strip().lower()
for host in (context.registry.allowed_hosts or [])
}
allowed_schemes = {
str(scheme).strip().lower()
for scheme in (context.registry.allowed_schemes or [])
}
violations = []
logger.reason(
@@ -68,7 +74,10 @@ class NoExternalEndpointsStage:
if violations:
return StageExecutionResult(
decision=ComplianceDecision.BLOCKED,
details_json={"endpoint_count": len(endpoints), "violations_count": len(violations)},
details_json={
"endpoint_count": len(endpoints),
"violations_count": len(violations),
},
violations=violations,
)
@@ -77,6 +86,8 @@ class NoExternalEndpointsStage:
details_json={"endpoint_count": len(endpoints), "violations_count": 0},
violations=[],
)
# [/DEF:NoExternalEndpointsStage:Class]
# [/DEF:backend.src.services.clean_release.stages.no_external_endpoints:Module]
# [/DEF:no_external_endpoints:Module]

118
backend/src/services/health_service.py
View File

@@ -3,14 +3,15 @@
# @SEMANTICS: health, aggregation, dashboards
# @PURPOSE: Business logic for aggregating dashboard health status from validation records.
# @LAYER: Domain/Service
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.cleanup.TaskCleanupService]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[SupersetClient]
# @RELATION: [DEPENDS_ON] ->[TaskCleanupService]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
from typing import List, Dict, Any, Optional, Tuple
from typing import List, Dict, Any, Optional, Tuple, cast
import time
from sqlalchemy.orm import Session
from sqlalchemy import func, desc
from sqlalchemy import func
import os
from ..models.llm import ValidationRecord
from ..schemas.health import DashboardHealthItem, HealthSummaryResponse
@@ -20,6 +21,10 @@ from ..core.task_manager.cleanup import TaskCleanupService
from ..core.task_manager import TaskManager
def _empty_dashboard_meta() -> Dict[str, Optional[str]]:
return cast(Dict[str, Optional[str]], {"slug": None, "title": None})
# [DEF:HealthService:Class]
# @COMPLEXITY: 4
# @PURPOSE: Aggregate latest dashboard validation state and manage persisted health report lifecycle.
@@ -27,12 +32,12 @@ from ..core.task_manager import TaskManager
# @POST: Exposes health summary aggregation and validation report deletion operations.
# @SIDE_EFFECT: Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators.
# @DATA_CONTRACT: Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool]
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.DashboardHealthItem]
# @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.HealthSummaryResponse]
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[DashboardHealthItem]
# @RELATION: [DEPENDS_ON] ->[HealthSummaryResponse]
# @RELATION: [DEPENDS_ON] ->[SupersetClient]
# @RELATION: [DEPENDS_ON] ->[TaskCleanupService]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
class HealthService:
_dashboard_summary_cache: Dict[
str, Tuple[float, Dict[str, Dict[str, Optional[str]]]]
@@ -50,8 +55,7 @@ class HealthService:
# @POST: Service is ready to aggregate summaries and delete health reports.
# @SIDE_EFFECT: Initializes per-instance dashboard metadata cache.
# @DATA_CONTRACT: Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService]
# @RELATION: [BINDS_TO] ->[backend.src.services.health_service.HealthService]
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
# @RELATION: [BINDS_TO] ->[HealthService]
def __init__(self, db: Session, config_manager=None):
self.db = db
self.config_manager = config_manager
@@ -66,10 +70,9 @@ class HealthService:
# @POST: Numeric dashboard ids for known environments are cached when discoverable.
# @SIDE_EFFECT: May call Superset dashboard list API once per referenced environment.
# @DATA_CONTRACT: Input[records: List[ValidationRecord]] -> Output[None]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [CALLS] ->[config_manager.get_environments]
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_dashboards_summary]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[SupersetClient]
def _prime_dashboard_meta_cache(self, records: List[ValidationRecord]) -> None:
if not self.config_manager or not records:
return
@@ -98,23 +101,26 @@ class HealthService:
env = environments.get(environment_id)
if not env:
for dashboard_id in dashboard_ids:
self._dashboard_meta_cache[(environment_id, dashboard_id)] = {
"slug": None,
"title": None,
}
self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
_empty_dashboard_meta()
)
continue
try:
cached_meta = self.__class__._dashboard_summary_cache.get(
environment_id
)
cache_is_fresh = (
dashboard_meta_map: Dict[str, Dict[str, Optional[str]]]
if (
cached_meta is not None
and (time.monotonic() - cached_meta[0])
< self.__class__._dashboard_summary_cache_ttl_seconds
)
if cache_is_fresh:
dashboard_meta_map = cached_meta[1]
):
cached_meta_data = cast(
Tuple[float, Dict[str, Dict[str, Optional[str]]]],
cached_meta,
)
dashboard_meta_map = cached_meta_data[1]
else:
dashboards = SupersetClient(env).get_dashboards_summary()
dashboard_meta_map = {
@@ -133,7 +139,7 @@ class HealthService:
self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
dashboard_meta_map.get(
dashboard_id,
{"slug": None, "title": None},
_empty_dashboard_meta(),
)
)
except Exception as exc:
@@ -143,10 +149,9 @@ class HealthService:
exc,
)
for dashboard_id in dashboard_ids:
self._dashboard_meta_cache[(environment_id, dashboard_id)] = {
"slug": None,
"title": None,
}
self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
_empty_dashboard_meta()
)
# [/DEF:_prime_dashboard_meta_cache:Function]
@@ -162,20 +167,20 @@ class HealthService:
normalized_dashboard_id = str(dashboard_id or "").strip()
normalized_environment_id = str(environment_id or "").strip()
if not normalized_dashboard_id:
return {"slug": None, "title": None}
return _empty_dashboard_meta()
if not normalized_dashboard_id.isdigit():
return {"slug": normalized_dashboard_id, "title": None}
if not self.config_manager or not normalized_environment_id:
return {"slug": None, "title": None}
return _empty_dashboard_meta()
cache_key = (normalized_environment_id, normalized_dashboard_id)
cached = self._dashboard_meta_cache.get(cache_key)
if cached is not None:
return cached
meta = {"slug": None, "title": None}
meta = _empty_dashboard_meta()
self._dashboard_meta_cache[cache_key] = meta
return meta
@@ -188,10 +193,10 @@ class HealthService:
# @POST: Returns HealthSummaryResponse with counts and latest record row per dashboard.
# @SIDE_EFFECT: May call Superset API to resolve dashboard metadata.
# @DATA_CONTRACT: Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse]
# @RELATION: [CALLS] ->[self._prime_dashboard_meta_cache]
# @RELATION: [CALLS] ->[self._resolve_dashboard_meta]
# @RELATION: [CALLS] ->[_prime_dashboard_meta_cache]
# @RELATION: [CALLS] ->[_resolve_dashboard_meta]
async def get_health_summary(
self, environment_id: str = None
self, environment_id: str = ""
) -> HealthSummaryResponse:
"""
@PURPOSE: Aggregates the latest validation status for all dashboards.
@@ -228,7 +233,8 @@ class HealthService:
unknown_count = 0
for rec in records:
status = rec.status.upper()
record = cast(Any, rec)
status = str(record.status or "").upper()
if status == "PASS":
pass_count += 1
elif status == "WARN":
@@ -239,18 +245,34 @@ class HealthService:
unknown_count += 1
status = "UNKNOWN"
meta = self._resolve_dashboard_meta(rec.dashboard_id, rec.environment_id)
record_id = str(record.id or "")
dashboard_id = str(record.dashboard_id or "")
resolved_environment_id = (
str(record.environment_id)
if record.environment_id is not None
else None
)
response_environment_id = (
resolved_environment_id
if resolved_environment_id is not None
else "unknown"
)
task_id = str(record.task_id) if record.task_id is not None else None
summary = str(record.summary) if record.summary is not None else None
timestamp = cast(Any, record.timestamp)
meta = self._resolve_dashboard_meta(dashboard_id, resolved_environment_id)
items.append(
DashboardHealthItem(
record_id=rec.id,
dashboard_id=rec.dashboard_id,
record_id=record_id,
dashboard_id=dashboard_id,
dashboard_slug=meta.get("slug"),
dashboard_title=meta.get("title"),
environment_id=rec.environment_id or "unknown",
environment_id=response_environment_id,
status=status,
last_check=rec.timestamp,
task_id=rec.task_id,
summary=rec.summary,
last_check=timestamp,
task_id=task_id,
summary=summary,
)
)
@@ -275,11 +297,9 @@ class HealthService:
# @POST: Returns True only when a matching record was deleted.
# @SIDE_EFFECT: Deletes DB rows, optional screenshot file, and optional task/log persistence.
# @DATA_CONTRACT: Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.TaskManager]
# @RELATION: [CALLS] ->[os.path.exists]
# @RELATION: [CALLS] ->[os.remove]
# @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService.delete_task_with_logs]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TaskCleanupService]
def delete_validation_report(
self, record_id: str, task_manager: Optional[TaskManager] = None
) -> bool:

82
backend/src/services/llm_provider.py
View File

@@ -3,8 +3,9 @@
# @SEMANTICS: service, llm, provider, encryption
# @PURPOSE: Service for managing LLM provider configurations with encrypted API keys.
# @LAYER: Domain
# @RELATION: DEPENDS_ON ->[backend.src.core.database:Function]
# @RELATION: DEPENDS_ON ->[backend.src.models.llm:Function]
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
# @RELATION: [DEPENDS_ON] ->[EncryptionManager]
# @RELATION: [DEPENDS_ON] ->[LLMProviderConfig]
from typing import List, Optional, TYPE_CHECKING
from sqlalchemy.orm import Session
from ..models.llm import LLMProvider
@@ -17,6 +18,7 @@ if TYPE_CHECKING:
MASKED_API_KEY_PLACEHOLDER = "********"
# [DEF:_require_fernet_key:Function]
# @COMPLEXITY: 5
# @PURPOSE: Load and validate the Fernet key used for secret encryption.
@@ -24,28 +26,41 @@ MASKED_API_KEY_PLACEHOLDER = "********"
# @POST: Returns validated key bytes ready for Fernet initialization.
# @RELATION: DEPENDS_ON ->[backend.src.core.logger:Function]
# @SIDE_EFFECT: Emits belief-state logs for missing or invalid encryption configuration.
# @DATA_CONTRACT: Input[ENCRYPTION_KEY:str] -> Output[bytes]
# @INVARIANT: Encryption initialization never falls back to a hardcoded secret.
def _require_fernet_key() -> bytes:
with belief_scope("_require_fernet_key"):
raw_key = os.getenv("ENCRYPTION_KEY", "").strip()
if not raw_key:
logger.explore("Missing ENCRYPTION_KEY blocks EncryptionManager initialization")
logger.explore(
"Missing ENCRYPTION_KEY blocks EncryptionManager initialization"
)
raise RuntimeError("ENCRYPTION_KEY must be set")
key = raw_key.encode()
try:
Fernet(key)
except Exception as exc:
logger.explore("Invalid ENCRYPTION_KEY blocks EncryptionManager initialization")
logger.explore(
"Invalid ENCRYPTION_KEY blocks EncryptionManager initialization"
)
raise RuntimeError("ENCRYPTION_KEY must be a valid Fernet key") from exc
logger.reflect("Validated ENCRYPTION_KEY for EncryptionManager initialization")
return key
# [/DEF:_require_fernet_key:Function]
# [DEF:EncryptionManager:Class]
# @COMPLEXITY: 5
# @PURPOSE: Handles encryption and decryption of sensitive data like API keys.
# @RELATION: [CALLS] ->[_require_fernet_key]
# @PRE: ENCRYPTION_KEY is configured with a valid Fernet key before instantiation.
# @POST: Manager exposes reversible encrypt/decrypt operations for persisted secrets.
# @SIDE_EFFECT: Initializes Fernet cryptography state from process environment.
# @DATA_CONTRACT: Input[str] -> Output[str]
# @INVARIANT: Uses only a validated secret key from environment.
#
# @TEST_CONTRACT: EncryptionManagerModel ->
@@ -67,6 +82,7 @@ class EncryptionManager:
def __init__(self):
self.key = _require_fernet_key()
self.fernet = Fernet(self.key)
# [/DEF:EncryptionManager_init:Function]
# [DEF:encrypt:Function]
@@ -76,6 +92,7 @@ class EncryptionManager:
def encrypt(self, data: str) -> str:
with belief_scope("encrypt"):
return self.fernet.encrypt(data.encode()).decode()
# [/DEF:encrypt:Function]
# [DEF:decrypt:Function]
@@ -85,20 +102,29 @@ class EncryptionManager:
def decrypt(self, encrypted_data: str) -> str:
with belief_scope("decrypt"):
return self.fernet.decrypt(encrypted_data.encode()).decode()
# [/DEF:decrypt:Function]
# [/DEF:EncryptionManager:Class]
# [DEF:LLMProviderService:Class]
# @COMPLEXITY: 3
# @PURPOSE: Service to manage LLM provider lifecycle.
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
# @RELATION: [DEPENDS_ON] ->[EncryptionManager]
# @RELATION: [DEPENDS_ON] ->[LLMProviderConfig]
class LLMProviderService:
# [DEF:LLMProviderService_init:Function]
# @PURPOSE: Initialize the service with database session.
# @PRE: db must be a valid SQLAlchemy Session.
# @POST: Service ready for provider operations.
# @RELATION: [DEPENDS_ON] ->[EncryptionManager]
def __init__(self, db: Session):
self.db = db
self.encryption = EncryptionManager()
# [/DEF:LLMProviderService_init:Function]
# [DEF:get_all_providers:Function]
@@ -106,9 +132,11 @@ class LLMProviderService:
# @PURPOSE: Returns all configured LLM providers.
# @PRE: Database connection must be active.
# @POST: Returns list of all LLMProvider records.
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
def get_all_providers(self) -> List[LLMProvider]:
with belief_scope("get_all_providers"):
return self.db.query(LLMProvider).all()
# [/DEF:get_all_providers:Function]
# [DEF:get_provider:Function]
@@ -116,9 +144,13 @@ class LLMProviderService:
# @PURPOSE: Returns a single LLM provider by ID.
# @PRE: provider_id must be a valid string.
# @POST: Returns LLMProvider or None if not found.
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
def get_provider(self, provider_id: str) -> Optional[LLMProvider]:
with belief_scope("get_provider"):
return self.db.query(LLMProvider).filter(LLMProvider.id == provider_id).first()
return (
self.db.query(LLMProvider).filter(LLMProvider.id == provider_id).first()
)
# [/DEF:get_provider:Function]
# [DEF:create_provider:Function]
@@ -126,6 +158,9 @@ class LLMProviderService:
# @PURPOSE: Creates a new LLM provider with encrypted API key.
# @PRE: config must contain valid provider configuration.
# @POST: New provider created and persisted to database.
# @RELATION: [DEPENDS_ON] ->[LLMProviderConfig]
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
# @RELATION: [CALLS] ->[encrypt]
def create_provider(self, config: "LLMProviderConfig") -> LLMProvider:
with belief_scope("create_provider"):
encrypted_key = self.encryption.encrypt(config.api_key)
@@ -135,12 +170,13 @@ class LLMProviderService:
base_url=config.base_url,
api_key=encrypted_key,
default_model=config.default_model,
is_active=config.is_active
is_active=config.is_active,
)
self.db.add(db_provider)
self.db.commit()
self.db.refresh(db_provider)
return db_provider
# [/DEF:create_provider:Function]
# [DEF:update_provider:Function]
@@ -148,12 +184,17 @@ class LLMProviderService:
# @PURPOSE: Updates an existing LLM provider.
# @PRE: provider_id must exist, config must be valid.
# @POST: Provider updated and persisted to database.
def update_provider(self, provider_id: str, config: "LLMProviderConfig") -> Optional[LLMProvider]:
# @RELATION: [DEPENDS_ON] ->[LLMProviderConfig]
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
# @RELATION: [CALLS] ->[encrypt]
def update_provider(
self, provider_id: str, config: "LLMProviderConfig"
) -> Optional[LLMProvider]:
with belief_scope("update_provider"):
db_provider = self.get_provider(provider_id)
if not db_provider:
return None
db_provider.provider_type = config.provider_type.value
db_provider.name = config.name
db_provider.base_url = config.base_url
@@ -166,10 +207,11 @@ class LLMProviderService:
db_provider.api_key = self.encryption.encrypt(config.api_key)
db_provider.default_model = config.default_model
db_provider.is_active = config.is_active
self.db.commit()
self.db.refresh(db_provider)
return db_provider
# [/DEF:update_provider:Function]
# [DEF:delete_provider:Function]
@@ -177,6 +219,7 @@ class LLMProviderService:
# @PURPOSE: Deletes an LLM provider.
# @PRE: provider_id must exist.
# @POST: Provider removed from database.
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
def delete_provider(self, provider_id: str) -> bool:
with belief_scope("delete_provider"):
db_provider = self.get_provider(provider_id)
@@ -185,6 +228,7 @@ class LLMProviderService:
self.db.delete(db_provider)
self.db.commit()
return True
# [/DEF:delete_provider:Function]
# [DEF:get_decrypted_api_key:Function]
@@ -192,25 +236,35 @@ class LLMProviderService:
# @PURPOSE: Returns the decrypted API key for a provider.
# @PRE: provider_id must exist with valid encrypted key.
# @POST: Returns decrypted API key or None on failure.
# @RELATION: [DEPENDS_ON] ->[LLMProvider]
# @RELATION: [CALLS] ->[decrypt]
def get_decrypted_api_key(self, provider_id: str) -> Optional[str]:
with belief_scope("get_decrypted_api_key"):
db_provider = self.get_provider(provider_id)
if not db_provider:
logger.warning(f"[get_decrypted_api_key] Provider {provider_id} not found in database")
logger.warning(
f"[get_decrypted_api_key] Provider {provider_id} not found in database"
)
return None
logger.info(f"[get_decrypted_api_key] Provider found: {db_provider.id}")
logger.info(f"[get_decrypted_api_key] Encrypted API key length: {len(db_provider.api_key) if db_provider.api_key else 0}")
logger.info(
f"[get_decrypted_api_key] Encrypted API key length: {len(db_provider.api_key) if db_provider.api_key else 0}"
)
try:
decrypted_key = self.encryption.decrypt(db_provider.api_key)
logger.info(f"[get_decrypted_api_key] Decryption successful, key length: {len(decrypted_key) if decrypted_key else 0}")
logger.info(
f"[get_decrypted_api_key] Decryption successful, key length: {len(decrypted_key) if decrypted_key else 0}"
)
return decrypted_key
except Exception as e:
logger.error(f"[get_decrypted_api_key] Decryption failed: {str(e)}")
return None
# [/DEF:get_decrypted_api_key:Function]
# [/DEF:LLMProviderService:Class]
# [/DEF:llm_provider:Module]

156
backend/src/services/notifications/service.py
View File

@@ -4,9 +4,13 @@
# @SEMANTICS: notifications, service, routing, dispatch, background-tasks
# @PURPOSE: Orchestrates notification routing based on user preferences and policy context.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.services.notifications.providers
# @RELATION: DEPENDS_ON -> backend.src.services.profile_service
# @RELATION: DEPENDS_ON -> backend.src.models.llm
# @RELATION: [DEPENDS_ON] ->[NotificationProvider]
# @RELATION: [DEPENDS_ON] ->[SMTPProvider]
# @RELATION: [DEPENDS_ON] ->[TelegramProvider]
# @RELATION: [DEPENDS_ON] ->[SlackProvider]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[ValidationPolicy]
# @RELATION: [DEPENDS_ON] ->[UserDashboardPreference]
#
# @INVARIANT: Notifications are dispatched asynchronously via BackgroundTasks.
# @INVARIANT: Missing profile or provider config must not crash the pipeline.
@@ -19,98 +23,165 @@ from ...core.logger import logger, belief_scope
from ...core.config_manager import ConfigManager
from ...models.llm import ValidationRecord, ValidationPolicy
from ...models.profile import UserDashboardPreference
from .providers import SMTPProvider, TelegramProvider, SlackProvider, NotificationProvider
from .providers import (
SMTPProvider,
TelegramProvider,
SlackProvider,
NotificationProvider,
)
# [DEF:NotificationService:Class]
# @PURPOSE: Routes validation reports to appropriate users and channels.
# @COMPLEXITY: 4
# @RELATION: [DEPENDS_ON] ->[NotificationProvider]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[ValidationPolicy]
# @RELATION: [DEPENDS_ON] ->[UserDashboardPreference]
# @PRE: Service receives a live DB session and configuration manager with notification payload settings.
# @POST: Service can resolve targets and dispatch provider sends without mutating validation records.
# @SIDE_EFFECT: Reads notification configuration, queries user preferences, and dispatches provider I/O.
# @DATA_CONTRACT: Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None]
class NotificationService:
# [DEF:NotificationService_init:Function]
# @COMPLEXITY: 3
# @PURPOSE: Bind DB and configuration collaborators used for provider initialization and routing.
# @RELATION: [BINDS_TO] ->[NotificationService]
# @RELATION: [DEPENDS_ON] ->[ValidationPolicy]
def __init__(self, db: Session, config_manager: ConfigManager):
self.db = db
self.config_manager = config_manager
self._providers: Dict[str, NotificationProvider] = {}
self._initialized = False
# [/DEF:NotificationService_init:Function]
# [DEF:_initialize_providers:Function]
# @COMPLEXITY: 3
# @PURPOSE: Materialize configured notification channel adapters once per service lifetime.
# @RELATION: [DEPENDS_ON] ->[SMTPProvider]
# @RELATION: [DEPENDS_ON] ->[TelegramProvider]
# @RELATION: [DEPENDS_ON] ->[SlackProvider]
def _initialize_providers(self):
if self._initialized:
return
# In a real implementation, we would fetch these from NotificationConfig model
# For now, we'll use a placeholder initialization logic
# T033 will implement the UI/API for this.
configs = self.config_manager.get_payload().get("notifications", {})
if "smtp" in configs:
self._providers["SMTP"] = SMTPProvider(configs["smtp"])
if "telegram" in configs:
self._providers["TELEGRAM"] = TelegramProvider(configs["telegram"])
if "slack" in configs:
self._providers["SLACK"] = SlackProvider(configs["slack"])
self._initialized = True
# [/DEF:_initialize_providers:Function]
# [DEF:dispatch_report:Function]
# @COMPLEXITY: 4
# @PURPOSE: Route one validation record to resolved owners and configured custom channels.
# @RELATION: [CALLS] ->[_initialize_providers]
# @RELATION: [CALLS] ->[_should_notify]
# @RELATION: [CALLS] ->[_resolve_targets]
# @RELATION: [CALLS] ->[_build_body]
# @PRE: record is persisted and providers can be initialized from configuration payload.
# @POST: Eligible notification sends are scheduled in background or awaited inline.
# @SIDE_EFFECT: Schedules or performs outbound provider sends and emits notification logs.
# @DATA_CONTRACT: Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None]
async def dispatch_report(
self,
record: ValidationRecord,
self,
record: ValidationRecord,
policy: Optional[ValidationPolicy] = None,
background_tasks: Optional[BackgroundTasks] = None
background_tasks: Optional[BackgroundTasks] = None,
):
"""
Route a validation record to owners and custom channels.
@PRE: record is persisted.
@POST: Dispatches async tasks for each resolved target.
"""
with belief_scope("NotificationService.dispatch_report", f"record_id={record.id}"):
with belief_scope(
"NotificationService.dispatch_report", f"record_id={record.id}"
):
self._initialize_providers()
# 1. Determine if we should notify based on status and policy
should_notify = self._should_notify(record, policy)
if not should_notify:
logger.reason(f"[REASON] Notification skipped for record {record.id} (status={record.status})")
logger.reason(
f"[REASON] Notification skipped for record {record.id} (status={record.status})"
)
return
# 2. Resolve targets (Owners + Custom Channels)
targets = self._resolve_targets(record, policy)
# 3. Dispatch
subject = f"Dashboard Health Alert: {record.status}"
body = self._build_body(record)
for channel_type, recipient in targets:
provider = self._providers.get(channel_type)
if not provider:
logger.warning(f"[NotificationService][EXPLORE] Unsupported or unconfigured channel: {channel_type}")
logger.warning(
f"[NotificationService][EXPLORE] Unsupported or unconfigured channel: {channel_type}"
)
continue
if background_tasks:
background_tasks.add_task(provider.send, recipient, subject, body)
else:
# Fallback to sync for tests or if no background_tasks provided
await provider.send(recipient, subject, body)
def _should_notify(self, record: ValidationRecord, policy: Optional[ValidationPolicy]) -> bool:
# [/DEF:dispatch_report:Function]
# [DEF:_should_notify:Function]
# @COMPLEXITY: 3
# @PURPOSE: Evaluate record status against effective alert policy.
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[ValidationPolicy]
def _should_notify(
self, record: ValidationRecord, policy: Optional[ValidationPolicy]
) -> bool:
condition = policy.alert_condition if policy else "FAIL_ONLY"
if condition == "ALWAYS":
return True
if condition == "WARN_AND_FAIL":
return record.status in ("WARN", "FAIL")
return record.status == "FAIL"
def _resolve_targets(self, record: ValidationRecord, policy: Optional[ValidationPolicy]) -> List[tuple]:
# [/DEF:_should_notify:Function]
# [DEF:_resolve_targets:Function]
# @COMPLEXITY: 3
# @PURPOSE: Resolve owner and policy-defined delivery targets for one validation record.
# @RELATION: [CALLS] ->[_find_dashboard_owners]
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[ValidationPolicy]
def _resolve_targets(
self, record: ValidationRecord, policy: Optional[ValidationPolicy]
) -> List[tuple]:
targets = []
# Owner routing
if not policy or policy.notify_owners:
owners = self._find_dashboard_owners(record)
for owner_pref in owners:
if not owner_pref.notify_on_fail:
continue
if owner_pref.telegram_id:
targets.append(("TELEGRAM", owner_pref.telegram_id))
email = owner_pref.email_address or getattr(owner_pref.user, "email", None)
email = owner_pref.email_address or getattr(
owner_pref.user, "email", None
)
if email:
targets.append(("SMTP", email))
@@ -119,20 +190,37 @@ class NotificationService:
for channel in policy.custom_channels:
# channel format: {"type": "SLACK", "target": "#alerts"}
targets.append((channel.get("type"), channel.get("target")))
return targets
def _find_dashboard_owners(self, record: ValidationRecord) -> List[UserDashboardPreference]:
# This is a simplified owner lookup.
# [/DEF:_resolve_targets:Function]
# [DEF:_find_dashboard_owners:Function]
# @COMPLEXITY: 3
# @PURPOSE: Load candidate dashboard owners from persisted profile preferences.
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[UserDashboardPreference]
def _find_dashboard_owners(
self, record: ValidationRecord
) -> List[UserDashboardPreference]:
# This is a simplified owner lookup.
# In a real scenario, we'd query Superset for owners, then match them to our UserDashboardPreference.
# For now, we'll return all users who have bound this dashboard's environment and have a username.
# Placeholder: return all preferences that have a superset_username
# (In production, we'd filter by actual ownership from Superset metadata)
return self.db.query(UserDashboardPreference).filter(
UserDashboardPreference.superset_username != None
).all()
return (
self.db.query(UserDashboardPreference)
.filter(UserDashboardPreference.superset_username != None)
.all()
)
# [/DEF:_find_dashboard_owners:Function]
# [DEF:_build_body:Function]
# @COMPLEXITY: 2
# @PURPOSE: Format one validation record into provider-ready body text.
# @RELATION: [DEPENDS_ON] ->[ValidationRecord]
def _build_body(self, record: ValidationRecord) -> str:
return (
f"Dashboard ID: {record.dashboard_id}\n"
@@ -141,6 +229,10 @@ class NotificationService:
f"Summary: {record.summary}\n\n"
f"Issues found: {len(record.issues)}"
)
# [/DEF:_build_body:Function]
# [/DEF:NotificationService:Class]
# [/DEF:service:Module]
# [/DEF:service:Module]

90
backend/src/services/reports/report_service.py
View File

@@ -3,9 +3,13 @@
# @SEMANTICS: reports, service, aggregation, filtering, pagination, detail
# @PURPOSE: Aggregate, normalize, filter, and paginate task reports for unified list/detail API use cases.
# @LAYER: Domain
# @RELATION: DEPENDS_ON ->[backend.src.core.task_manager.manager.TaskManager:Function]
# @RELATION: DEPENDS_ON ->[backend.src.models.report:Function]
# @RELATION: DEPENDS_ON ->[backend.src.services.reports.normalizer:Function]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TaskReport]
# @RELATION: [DEPENDS_ON] ->[ReportQuery]
# @RELATION: [DEPENDS_ON] ->[ReportCollection]
# @RELATION: [DEPENDS_ON] ->[ReportDetailView]
# @RELATION: [DEPENDS_ON] ->[normalize_task_report]
# @RELATION: [DEPENDS_ON] ->[CleanReleaseRepository]
# @INVARIANT: List responses are deterministic and include applied filter echo metadata.
# [SECTION: IMPORTS]
@@ -15,7 +19,14 @@ from typing import List, Optional
from ...core.logger import belief_scope
from ...core.task_manager import TaskManager
from ...models.report import ReportCollection, ReportDetailView, ReportQuery, ReportStatus, TaskReport, TaskType
from ...models.report import (
ReportCollection,
ReportDetailView,
ReportQuery,
ReportStatus,
TaskReport,
TaskType,
)
from ..clean_release.repository import CleanReleaseRepository
from .normalizer import normalize_task_report
# [/SECTION]
@@ -26,6 +37,11 @@ from .normalizer import normalize_task_report
# @COMPLEXITY: 5
# @PRE: TaskManager dependency is initialized.
# @POST: Provides deterministic list/detail report responses.
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[CleanReleaseRepository]
# @RELATION: [CALLS] ->[normalize_task_report]
# @SIDE_EFFECT: Reads task history and optional clean-release repository state without mutating source records.
# @DATA_CONTRACT: Input[TaskManager, Optional[CleanReleaseRepository], ReportQuery|report_id] -> Output[ReportCollection|ReportDetailView|None]
# @INVARIANT: Service methods are read-only over task history source.
#
# @TEST_CONTRACT: ReportsServiceModel ->
@@ -47,11 +63,21 @@ class ReportsService:
# @PRE: task_manager is a live TaskManager instance.
# @POST: self.task_manager is assigned and ready for read operations.
# @INVARIANT: Constructor performs no task mutations.
# @RELATION: [BINDS_TO] ->[ReportsService]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[CleanReleaseRepository]
# @SIDE_EFFECT: Stores collaborator references for later read-only report projections.
# @DATA_CONTRACT: Input[TaskManager, Optional[CleanReleaseRepository]] -> Output[ReportsService]
# @PARAM: task_manager (TaskManager) - Task manager providing source task history.
def __init__(self, task_manager: TaskManager, clean_release_repository: Optional[CleanReleaseRepository] = None):
def __init__(
self,
task_manager: TaskManager,
clean_release_repository: Optional[CleanReleaseRepository] = None,
):
with belief_scope("__init__"):
self.task_manager = task_manager
self.clean_release_repository = clean_release_repository
# [/DEF:init:Function]
# [DEF:_load_normalized_reports:Function]
@@ -65,6 +91,7 @@ class ReportsService:
tasks = self.task_manager.get_all_tasks()
reports = [normalize_task_report(task) for task in tasks]
return reports
# [/DEF:_load_normalized_reports:Function]
# [DEF:_to_utc_datetime:Function]
@@ -81,6 +108,7 @@ class ReportsService:
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
# [/DEF:_to_utc_datetime:Function]
# [DEF:_datetime_sort_key:Function]
@@ -96,6 +124,7 @@ class ReportsService:
if updated is None:
return 0.0
return updated.timestamp()
# [/DEF:_datetime_sort_key:Function]
# [DEF:_matches_query:Function]
@@ -116,9 +145,17 @@ class ReportsService:
query_time_from = self._to_utc_datetime(query.time_from)
query_time_to = self._to_utc_datetime(query.time_to)
if query_time_from and report_updated_at and report_updated_at < query_time_from:
if (
query_time_from
and report_updated_at
and report_updated_at < query_time_from
):
return False
if query_time_to and report_updated_at and report_updated_at > query_time_to:
if (
query_time_to
and report_updated_at
and report_updated_at > query_time_to
):
return False
if query.search:
needle = query.search.lower()
@@ -126,6 +163,7 @@ class ReportsService:
if needle not in haystack:
return False
return True
# [/DEF:_matches_query:Function]
# [DEF:_sort_reports:Function]
@@ -136,7 +174,9 @@ class ReportsService:
# @PARAM: reports (List[TaskReport]) - Filtered reports.
# @PARAM: query (ReportQuery) - Sort config.
# @RETURN: List[TaskReport] - Sorted reports.
def _sort_reports(self, reports: List[TaskReport], query: ReportQuery) -> List[TaskReport]:
def _sort_reports(
self, reports: List[TaskReport], query: ReportQuery
) -> List[TaskReport]:
with belief_scope("_sort_reports"):
reverse = query.sort_order == "desc"
@@ -148,6 +188,7 @@ class ReportsService:
reports.sort(key=self._datetime_sort_key, reverse=reverse)
return reports
# [/DEF:_sort_reports:Function]
# [DEF:list_reports:Function]
@@ -159,7 +200,9 @@ class ReportsService:
def list_reports(self, query: ReportQuery) -> ReportCollection:
with belief_scope("list_reports"):
reports = self._load_normalized_reports()
filtered = [report for report in reports if self._matches_query(report, query)]
filtered = [
report for report in reports if self._matches_query(report, query)
]
sorted_reports = self._sort_reports(filtered, query)
total = len(sorted_reports)
@@ -176,6 +219,7 @@ class ReportsService:
has_next=has_next,
applied_filters=query,
)
# [/DEF:list_reports:Function]
# [DEF:get_report_detail:Function]
@@ -187,13 +231,17 @@ class ReportsService:
def get_report_detail(self, report_id: str) -> Optional[ReportDetailView]:
with belief_scope("get_report_detail"):
reports = self._load_normalized_reports()
target = next((report for report in reports if report.report_id == report_id), None)
target = next(
(report for report in reports if report.report_id == report_id), None
)
if not target:
return None
timeline = []
if target.started_at:
timeline.append({"event": "started", "at": target.started_at.isoformat()})
timeline.append(
{"event": "started", "at": target.started_at.isoformat()}
)
timeline.append({"event": "updated", "at": target.updated_at.isoformat()})
diagnostics = target.details or {}
@@ -202,12 +250,17 @@ class ReportsService:
if target.error_context:
diagnostics["error_context"] = target.error_context.model_dump()
if target.task_type == TaskType.CLEAN_RELEASE and self.clean_release_repository is not None:
if (
target.task_type == TaskType.CLEAN_RELEASE
and self.clean_release_repository is not None
):
run_id = None
if isinstance(diagnostics, dict):
result_payload = diagnostics.get("result")
if isinstance(result_payload, dict):
run_id = result_payload.get("run_id") or result_payload.get("check_run_id")
run_id = result_payload.get("run_id") or result_payload.get(
"check_run_id"
)
if run_id:
run = self.clean_release_repository.get_check_run(str(run_id))
if run is not None:
@@ -219,7 +272,11 @@ class ReportsService:
"requested_by": run.requested_by,
}
linked_report = next(
(item for item in self.clean_release_repository.reports.values() if item.run_id == run.id),
(
item
for item in self.clean_release_repository.reports.values()
if item.run_id == run.id
),
None,
)
if linked_report is not None:
@@ -240,7 +297,10 @@ class ReportsService:
diagnostics=diagnostics,
next_actions=next_actions,
)
# [/DEF:get_report_detail:Function]
# [/DEF:ReportsService:Class]
# [/DEF:report_service:Module]
# [/DEF:report_service:Module]

7
backend/tests/core/migration/test_archive_parser.py
View File

@@ -3,7 +3,7 @@
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for MigrationArchiveParser ZIP extraction contract.
# @LAYER: Domain
# @RELATION: VERIFIES -> backend.src.core.migration.archive_parser
# @RELATION: DEPENDS_ON -> [MigrationArchiveParserModule]
#
import os
import sys
@@ -23,6 +23,11 @@ from src.core.migration.archive_parser import MigrationArchiveParser
# [DEF:test_extract_objects_from_zip_collects_all_types:Function]
# @RELATION: BINDS_TO -> TestArchiveParser
# @PURPOSE: Verify archive parser collects dashboard/chart/dataset YAML objects into typed buckets.
# @TEST_CONTRACT: zip_archive_fixture -> typed dashboard/chart/dataset extraction buckets
# @TEST_SCENARIO: archive_with_supported_objects_extracts_all_types -> One YAML file per supported type lands in matching bucket.
# @TEST_EDGE: missing_field -> Missing typed folder prevents that object class from being extracted.
# @TEST_EDGE: invalid_type -> Unsupported file layout is ignored by typed extraction buckets.
# @TEST_EDGE: external_fail -> Corrupt archive would fail extraction before typed buckets are returned.
def test_extract_objects_from_zip_collects_all_types():
parser = MigrationArchiveParser()
with tempfile.TemporaryDirectory() as td:

7
backend/tests/core/migration/test_dry_run_orchestrator.py
View File

@@ -3,7 +3,7 @@
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for MigrationDryRunService diff and risk computation contracts.
# @LAYER: Domain
# @RELATION: VERIFIES -> [MigrationDryRunOrchestratorModule]
# @RELATION: DEPENDS_ON -> [MigrationDryRunOrchestratorModule]
#
import json
import sys
@@ -56,6 +56,11 @@ def _make_session():
# [DEF:test_migration_dry_run_service_builds_diff_and_risk:Function]
# @RELATION: BINDS_TO -> [TestDryRunOrchestrator]
# @PURPOSE: Verify dry-run orchestration returns stable diff summary and required risk codes.
# @TEST_SCENARIO: dry_run_builds_diff_and_risk -> Stable diff summary and required risk codes are returned.
# @TEST_EDGE: missing_field -> Missing target datasource remains visible in risk items.
# @TEST_EDGE: invalid_type -> Broken dataset reference remains visible in risk items.
# @TEST_EDGE: external_fail -> Engine transform stub failure would stop result production.
# @TEST_INVARIANT: dry_run_result_contract_matches_fixture -> VERIFIED_BY: [dry_run_builds_diff_and_risk]
def test_migration_dry_run_service_builds_diff_and_risk():
# @TEST_CONTRACT: dry_run_result_contract -> {
# required_fields: {diff: object, summary: object, risk: object},

86
backend/tests/test_resource_hubs.py
View File

@@ -1,5 +1,6 @@
# [DEF:TestResourceHubs:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @RELATION: DEPENDS_ON -> [DashboardsApi]
# @RELATION: DEPENDS_ON -> [DatasetsApi]
# @COMPLEXITY: 3
# @SEMANTICS: tests, resource-hubs, dashboards, datasets, pagination, api
# @PURPOSE: Contract tests for resource hub dashboards/datasets listing and pagination boundary validation.
@@ -17,14 +18,20 @@ from src.dependencies import (
client = TestClient(app)
# [DEF:test_dashboards_api:Test]
# @RELATION: BINDS_TO -> TestResourceHubs
# [DEF:test_dashboards_api:Block]
# @RELATION: BINDS_TO -> [TestResourceHubs]
# @PURPOSE: Verify GET /api/dashboards contract compliance
# @TEST: Valid env_id returns 200 and dashboard list
# @TEST: Invalid env_id returns 404
# @TEST: Search filter works
# @TEST_CONTRACT: dashboards_query -> dashboards payload or not_found response
# @TEST_SCENARIO: dashboards_env_found_returns_payload -> HTTP 200 returns normalized dashboards list.
# @TEST_SCENARIO: dashboards_unknown_env_returns_not_found -> HTTP 404 is returned for unknown env_id.
# @TEST_SCENARIO: dashboards_search_filters_results -> Search narrows payload to matching dashboard title.
# @TEST_INVARIANT: dashboards_route_contract_stays_observable -> VERIFIED_BY: [dashboards_env_found_returns_payload, dashboards_unknown_env_returns_not_found, dashboards_search_filters_results]
# [DEF:mock_deps:Function]
# @PURPOSE: Provide dependency override fixture for resource hub route tests.
# @RELATION: BINDS_TO -> [TestResourceHubs]
# @TEST_FIXTURE: resource_hub_overrides -> INLINE_JSON
@pytest.fixture
def mock_deps():
# @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass
@@ -89,8 +96,11 @@ def mock_deps():
app.dependency_overrides.clear()
# [/DEF:mock_deps:Function]
# [DEF:test_get_dashboards_success:Function]
# @RELATION: BINDS_TO -> test_dashboards_api
# @RELATION: BINDS_TO -> [test_dashboards_api]
# @PURPOSE: Verify dashboards endpoint returns 200 with expected dashboard payload fields.
def test_get_dashboards_success(mock_deps):
response = client.get("/api/dashboards?env_id=env1")
@@ -106,7 +116,7 @@ def test_get_dashboards_success(mock_deps):
# [DEF:test_get_dashboards_not_found:Function]
# @RELATION: BINDS_TO -> test_dashboards_api
# @RELATION: BINDS_TO -> [test_dashboards_api]
# @PURPOSE: Verify dashboards endpoint returns 404 for unknown environment identifier.
def test_get_dashboards_not_found(mock_deps):
response = client.get("/api/dashboards?env_id=invalid")
@@ -117,7 +127,7 @@ def test_get_dashboards_not_found(mock_deps):
# [DEF:test_get_dashboards_search:Function]
# @RELATION: BINDS_TO -> test_dashboards_api
# @RELATION: BINDS_TO -> [test_dashboards_api]
# @PURPOSE: Verify dashboards endpoint search filter returns matching subset.
def test_get_dashboards_search(mock_deps):
response = client.get("/api/dashboards?env_id=env1&search=Sales")
@@ -128,20 +138,22 @@ def test_get_dashboards_search(mock_deps):
# [/DEF:test_get_dashboards_search:Function]
# [/DEF:test_dashboards_api:Test]
# [/DEF:test_dashboards_api:Block]
# [DEF:test_datasets_api:Test]
# @RELATION: BINDS_TO -> TestResourceHubs
# [DEF:test_datasets_api:Block]
# @RELATION: BINDS_TO -> [TestResourceHubs]
# @PURPOSE: Verify GET /api/datasets contract compliance
# @TEST: Valid env_id returns 200 and dataset list
# @TEST: Invalid env_id returns 404
# @TEST: Search filter works
# @TEST: Negative - Service failure returns 503
# @TEST_CONTRACT: datasets_query -> datasets payload or error response
# @TEST_SCENARIO: datasets_env_found_returns_payload -> HTTP 200 returns normalized datasets list.
# @TEST_SCENARIO: datasets_unknown_env_returns_not_found -> HTTP 404 is returned for unknown env_id.
# @TEST_SCENARIO: datasets_search_filters_results -> Search narrows payload to matching dataset table.
# @TEST_SCENARIO: datasets_service_failure_returns_503 -> Backend fetch failure surfaces as HTTP 503.
# @TEST_INVARIANT: datasets_route_contract_stays_observable -> VERIFIED_BY: [datasets_env_found_returns_payload, datasets_unknown_env_returns_not_found, datasets_search_filters_results, datasets_service_failure_returns_503]
# [DEF:test_get_datasets_success:Function]
# @RELATION: BINDS_TO -> test_datasets_api
# @RELATION: BINDS_TO -> [test_datasets_api]
# @PURPOSE: Verify datasets endpoint returns 200 with mapped fields payload.
def test_get_datasets_success(mock_deps):
mock_deps["resource"].get_datasets_with_status = AsyncMock(
@@ -170,7 +182,7 @@ def test_get_datasets_success(mock_deps):
# [DEF:test_get_datasets_not_found:Function]
# @RELATION: BINDS_TO -> test_datasets_api
# @RELATION: BINDS_TO -> [test_datasets_api]
# @PURPOSE: Verify datasets endpoint returns 404 for unknown environment identifier.
def test_get_datasets_not_found(mock_deps):
response = client.get("/api/datasets?env_id=invalid")
@@ -181,7 +193,7 @@ def test_get_datasets_not_found(mock_deps):
# [DEF:test_get_datasets_search:Function]
# @RELATION: BINDS_TO -> test_datasets_api
# @RELATION: BINDS_TO -> [test_datasets_api]
# @PURPOSE: Verify datasets endpoint search filter returns matching dataset subset.
def test_get_datasets_search(mock_deps):
mock_deps["resource"].get_datasets_with_status = AsyncMock(
@@ -216,7 +228,7 @@ def test_get_datasets_search(mock_deps):
# [DEF:test_get_datasets_service_failure:Function]
# @RELATION: BINDS_TO -> test_datasets_api
# @RELATION: BINDS_TO -> [test_datasets_api]
# @PURPOSE: Verify datasets endpoint surfaces backend fetch failure as HTTP 503.
def test_get_datasets_service_failure(mock_deps):
mock_deps["resource"].get_datasets_with_status = AsyncMock(
@@ -229,20 +241,28 @@ def test_get_datasets_service_failure(mock_deps):
# [/DEF:test_get_datasets_service_failure:Function]
# [/DEF:test_datasets_api:Test]
# [/DEF:test_datasets_api:Block]
# [DEF:test_pagination_boundaries:Test]
# @RELATION: BINDS_TO -> TestResourceHubs
# [DEF:test_pagination_boundaries:Block]
# @RELATION: BINDS_TO -> [TestResourceHubs]
# @PURPOSE: Verify pagination validation for GET endpoints
# @TEST: page<1 and page_size>100 return 400
# @TEST_CONTRACT: pagination_query -> validation error response
# @TEST_SCENARIO: dashboards_zero_page_rejected -> page=0 returns HTTP 400.
# @TEST_SCENARIO: dashboards_oversize_page_rejected -> page_size=101 returns HTTP 400.
# @TEST_SCENARIO: datasets_zero_page_rejected -> page=0 returns HTTP 400.
# @TEST_SCENARIO: datasets_oversize_page_rejected -> page_size=101 returns HTTP 400.
# @TEST_EDGE: missing_field -> Missing env_id prevents route contract completion.
# @TEST_EDGE: invalid_type -> Invalid pagination values are rejected at route validation layer.
# @TEST_EDGE: external_fail -> Validation failure returns HTTP 400 without partial payload.
# @TEST_INVARIANT: pagination_limits_apply_to_both_routes -> VERIFIED_BY: [dashboards_zero_page_rejected, dashboards_oversize_page_rejected, datasets_zero_page_rejected, datasets_oversize_page_rejected]
# [DEF:test_get_dashboards_pagination_zero_page:Function]
# @RELATION: BINDS_TO -> test_pagination_boundaries
# @RELATION: BINDS_TO -> [test_pagination_boundaries]
# @PURPOSE: Verify dashboards endpoint rejects page=0 with HTTP 400 validation error.
def test_get_dashboards_pagination_zero_page(mock_deps):
"""@TEST_EDGE: pagination_zero_page -> {page:0, status:400}"""
# @TEST_EDGE: pagination_zero_page -> {page: 0, status: 400}
response = client.get("/api/dashboards?env_id=env1&page=0")
assert response.status_code == 400
assert "Page must be >= 1" in response.json()["detail"]
@@ -252,10 +272,10 @@ def test_get_dashboards_pagination_zero_page(mock_deps):
# [DEF:test_get_dashboards_pagination_oversize:Function]
# @RELATION: BINDS_TO -> test_pagination_boundaries
# @RELATION: BINDS_TO -> [test_pagination_boundaries]
# @PURPOSE: Verify dashboards endpoint rejects oversized page_size with HTTP 400.
def test_get_dashboards_pagination_oversize(mock_deps):
"""@TEST_EDGE: pagination_oversize -> {page_size:101, status:400}"""
# @TEST_EDGE: pagination_oversize -> {page_size: 101, status: 400}
response = client.get("/api/dashboards?env_id=env1&page_size=101")
assert response.status_code == 400
assert "Page size must be between 1 and 100" in response.json()["detail"]
@@ -265,10 +285,10 @@ def test_get_dashboards_pagination_oversize(mock_deps):
# [DEF:test_get_datasets_pagination_zero_page:Function]
# @RELATION: BINDS_TO -> test_pagination_boundaries
# @RELATION: BINDS_TO -> [test_pagination_boundaries]
# @PURPOSE: Verify datasets endpoint rejects page=0 with HTTP 400.
def test_get_datasets_pagination_zero_page(mock_deps):
"""@TEST_EDGE: pagination_zero_page on datasets"""
# @TEST_EDGE: pagination_zero_page_datasets -> {page: 0, status: 400}
response = client.get("/api/datasets?env_id=env1&page=0")
assert response.status_code == 400
@@ -277,14 +297,14 @@ def test_get_datasets_pagination_zero_page(mock_deps):
# [DEF:test_get_datasets_pagination_oversize:Function]
# @RELATION: BINDS_TO -> test_pagination_boundaries
# @RELATION: BINDS_TO -> [test_pagination_boundaries]
# @PURPOSE: Verify datasets endpoint rejects oversized page_size with HTTP 400.
def test_get_datasets_pagination_oversize(mock_deps):
"""@TEST_EDGE: pagination_oversize on datasets"""
# @TEST_EDGE: pagination_oversize_datasets -> {page_size: 101, status: 400}
response = client.get("/api/datasets?env_id=env1&page_size=101")
assert response.status_code == 400
# [/DEF:test_get_datasets_pagination_oversize:Function]
# [/DEF:test_pagination_boundaries:Test]
# [/DEF:test_pagination_boundaries:Block]
# [/DEF:TestResourceHubs:Module]