busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

semantics

Browse Source
This commit is contained in:
busya
2026-03-27 21:27:31 +03:00
parent 7c85552132
commit 2ed66bfebc
182 changed files with 21186 additions and 10254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/scripts/__init__.py
View File

@@ -1,3 +1,3 @@
# [DEF:src.scripts:Package]
# [DEF:ScriptsPackage:Package]
# @PURPOSE: Script entrypoint package root.
# [/DEF:src.scripts:Package]
# [/DEF:ScriptsPackage:Package]

88
backend/src/scripts/clean_release_cli.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.scripts.clean_release_cli:Module]
# [DEF:CleanReleaseCliScript:Module]
# @COMPLEXITY: 3
# @SEMANTICS: cli, clean-release, candidate, artifacts, manifest
# @PURPOSE: Provide headless CLI commands for candidate registration, artifact import and manifest build.
@@ -12,10 +12,18 @@ from datetime import date, datetime, timezone
from typing import Any, Dict, List, Optional
from ..models.clean_release import CandidateArtifact, ReleaseCandidate
from ..services.clean_release.approval_service import approve_candidate, reject_candidate
from ..services.clean_release.compliance_execution_service import ComplianceExecutionService
from ..services.clean_release.approval_service import (
approve_candidate,
reject_candidate,
)
from ..services.clean_release.compliance_execution_service import (
ComplianceExecutionService,
)
from ..services.clean_release.enums import CandidateStatus
from ..services.clean_release.publication_service import publish_candidate, revoke_publication
from ..services.clean_release.publication_service import (
publish_candidate,
revoke_publication,
)
# [DEF:build_parser:Function]
@@ -88,6 +96,8 @@ def build_parser() -> argparse.ArgumentParser:
revoke.add_argument("--json", action="store_true")
return parser
# [/DEF:build_parser:Function]
@@ -97,6 +107,7 @@ def build_parser() -> argparse.ArgumentParser:
# @POST: Candidate is persisted in DRAFT status.
def run_candidate_register(args: argparse.Namespace) -> int:
from ..dependencies import get_clean_release_repository
repository = get_clean_release_repository()
existing = repository.get_candidate(args.candidate_id)
if existing is not None:
@@ -114,6 +125,8 @@ def run_candidate_register(args: argparse.Namespace) -> int:
repository.save_candidate(candidate)
print(json.dumps({"status": "ok", "candidate_id": candidate.id}))
return 0
# [/DEF:run_candidate_register:Function]
@@ -123,6 +136,7 @@ def run_candidate_register(args: argparse.Namespace) -> int:
# @POST: Artifact is persisted for candidate.
def run_artifact_import(args: argparse.Namespace) -> int:
from ..dependencies import get_clean_release_repository
repository = get_clean_release_repository()
candidate = repository.get_candidate(args.candidate_id)
if candidate is None:
@@ -144,6 +158,8 @@ def run_artifact_import(args: argparse.Namespace) -> int:
print(json.dumps({"status": "ok", "artifact_id": artifact.id}))
return 0
# [/DEF:run_artifact_import:Function]
@@ -166,8 +182,18 @@ def run_manifest_build(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": str(exc)}))
return 1
print(json.dumps({"status": "ok", "manifest_id": manifest.id, "version": manifest.manifest_version}))
print(
json.dumps(
{
"status": "ok",
"manifest_id": manifest.id,
"version": manifest.manifest_version,
}
)
)
return 0
# [/DEF:run_manifest_build:Function]
@@ -180,7 +206,9 @@ def run_compliance_run(args: argparse.Namespace) -> int:
repository = get_clean_release_repository()
config_manager = get_config_manager()
service = ComplianceExecutionService(repository=repository, config_manager=config_manager)
service = ComplianceExecutionService(
repository=repository, config_manager=config_manager
)
try:
result = service.execute_run(
@@ -203,6 +231,8 @@ def run_compliance_run(args: argparse.Namespace) -> int:
}
print(json.dumps(payload))
return 0
# [/DEF:run_compliance_run:Function]
@@ -219,7 +249,9 @@ def run_compliance_status(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": "run not found"}))
return 2
report = next((item for item in repository.reports.values() if item.run_id == run.id), None)
report = next(
(item for item in repository.reports.values() if item.run_id == run.id), None
)
payload = {
"status": "ok",
"run_id": run.id,
@@ -231,6 +263,8 @@ def run_compliance_status(args: argparse.Namespace) -> int:
}
print(json.dumps(payload))
return 0
# [/DEF:run_compliance_status:Function]
@@ -259,6 +293,8 @@ def _to_payload(value: Any) -> Dict[str, Any]:
row = {column.name: getattr(value, column.name) for column in table.columns}
return _normalize(row)
raise TypeError(f"unsupported payload type: {type(value)!r}")
# [/DEF:_to_payload:Function]
@@ -275,13 +311,17 @@ def run_compliance_report(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": "run not found"}))
return 2
report = next((item for item in repository.reports.values() if item.run_id == run.id), None)
report = next(
(item for item in repository.reports.values() if item.run_id == run.id), None
)
if report is None:
print(json.dumps({"status": "error", "message": "report not found"}))
return 2
print(json.dumps({"status": "ok", "report": _to_payload(report)}))
return 0
# [/DEF:run_compliance_report:Function]
@@ -299,8 +339,14 @@ def run_compliance_violations(args: argparse.Namespace) -> int:
return 2
violations = repository.get_violations_by_run(args.run_id)
print(json.dumps({"status": "ok", "items": [_to_payload(item) for item in violations]}))
print(
json.dumps(
{"status": "ok", "items": [_to_payload(item) for item in violations]}
)
)
return 0
# [/DEF:run_compliance_violations:Function]
@@ -324,8 +370,14 @@ def run_approve(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": str(exc)}))
return 2
print(json.dumps({"status": "ok", "decision": decision.decision, "decision_id": decision.id}))
print(
json.dumps(
{"status": "ok", "decision": decision.decision, "decision_id": decision.id}
)
)
return 0
# [/DEF:run_approve:Function]
@@ -349,8 +401,14 @@ def run_reject(args: argparse.Namespace) -> int:
print(json.dumps({"status": "error", "message": str(exc)}))
return 2
print(json.dumps({"status": "ok", "decision": decision.decision, "decision_id": decision.id}))
print(
json.dumps(
{"status": "ok", "decision": decision.decision, "decision_id": decision.id}
)
)
return 0
# [/DEF:run_reject:Function]
@@ -377,6 +435,8 @@ def run_publish(args: argparse.Namespace) -> int:
print(json.dumps({"status": "ok", "publication": _to_payload(publication)}))
return 0
# [/DEF:run_publish:Function]
@@ -401,6 +461,8 @@ def run_revoke(args: argparse.Namespace) -> int:
print(json.dumps({"status": "ok", "publication": _to_payload(publication)}))
return 0
# [/DEF:run_revoke:Function]
@@ -435,10 +497,12 @@ def main(argv: Optional[List[str]] = None) -> int:
print(json.dumps({"status": "error", "message": "unknown command"}))
return 2
# [/DEF:main:Function]
if __name__ == "__main__":
raise SystemExit(main())
# [/DEF:backend.src.scripts.clean_release_cli:Module]
# [/DEF:CleanReleaseCliScript:Module]

182
backend/src/scripts/clean_release_tui.py
View File

@@ -1,10 +1,10 @@
# [DEF:backend.src.scripts.clean_release_tui:Module]
# [DEF:CleanReleaseTuiScript:Module]
# @COMPLEXITY: 3
# @SEMANTICS: clean-release, tui, ncurses, interactive-validator
# @PURPOSE: Interactive terminal interface for Enterprise Clean Release compliance validation.
# @LAYER: UI
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.compliance_orchestrator
# @RELATION: DEPENDS_ON -> backend.src.services.clean_release.repository
# @RELATION: DEPENDS_ON -> [compliance_orchestrator]
# @RELATION: DEPENDS_ON -> [repository]
# @INVARIANT: TUI refuses startup in non-TTY environments; headless flow is CLI/API only.
import curses
@@ -37,12 +37,15 @@ from src.models.clean_release import (
)
from src.services.clean_release.approval_service import approve_candidate
from src.services.clean_release.artifact_catalog_loader import load_bootstrap_artifacts
from src.services.clean_release.compliance_execution_service import ComplianceExecutionService
from src.services.clean_release.compliance_execution_service import (
ComplianceExecutionService,
)
from src.services.clean_release.enums import CandidateStatus
from src.services.clean_release.manifest_service import build_manifest_snapshot
from src.services.clean_release.publication_service import publish_candidate
from src.services.clean_release.repository import CleanReleaseRepository
# [DEF:TuiFacadeAdapter:Class]
# @PURPOSE: Thin TUI adapter that routes business mutations through application services.
# @PRE: repository contains candidate and trusted policy/registry snapshots for execution.
@@ -67,15 +70,25 @@ class TuiFacadeAdapter:
manifests = self.repository.get_manifests_by_candidate(candidate_id)
if not manifests:
raise ValueError("Manifest required before compliance run")
latest_manifest = sorted(manifests, key=lambda item: item.manifest_version, reverse=True)[0]
latest_manifest = sorted(
manifests, key=lambda item: item.manifest_version, reverse=True
)[0]
service = ComplianceExecutionService(
repository=self.repository,
config_manager=self._build_config_manager(),
)
return service.execute_run(candidate_id=candidate_id, requested_by=actor, manifest_id=latest_manifest.id)
return service.execute_run(
candidate_id=candidate_id,
requested_by=actor,
manifest_id=latest_manifest.id,
)
def approve_latest(self, *, candidate_id: str, actor: str):
reports = [item for item in self.repository.reports.values() if item.candidate_id == candidate_id]
reports = [
item
for item in self.repository.reports.values()
if item.candidate_id == candidate_id
]
if not reports:
raise ValueError("No compliance report available for approval")
report = sorted(reports, key=lambda item: item.generated_at, reverse=True)[0]
@@ -88,7 +101,11 @@ class TuiFacadeAdapter:
)
def publish_latest(self, *, candidate_id: str, actor: str):
reports = [item for item in self.repository.reports.values() if item.candidate_id == candidate_id]
reports = [
item
for item in self.repository.reports.values()
if item.candidate_id == candidate_id
]
if not reports:
raise ValueError("No compliance report available for publication")
report = sorted(reports, key=lambda item: item.generated_at, reverse=True)[0]
@@ -111,24 +128,55 @@ class TuiFacadeAdapter:
def get_overview(self, *, candidate_id: str) -> Dict[str, Any]:
candidate = self.repository.get_candidate(candidate_id)
manifests = self.repository.get_manifests_by_candidate(candidate_id)
latest_manifest = sorted(manifests, key=lambda item: item.manifest_version, reverse=True)[0] if manifests else None
runs = [item for item in self.repository.check_runs.values() if item.candidate_id == candidate_id]
latest_run = sorted(runs, key=lambda item: item.requested_at, reverse=True)[0] if runs else None
latest_report = next((item for item in self.repository.reports.values() if latest_run and item.run_id == latest_run.id), None)
latest_manifest = (
sorted(manifests, key=lambda item: item.manifest_version, reverse=True)[0]
if manifests
else None
)
runs = [
item
for item in self.repository.check_runs.values()
if item.candidate_id == candidate_id
]
latest_run = (
sorted(runs, key=lambda item: item.requested_at, reverse=True)[0]
if runs
else None
)
latest_report = next(
(
item
for item in self.repository.reports.values()
if latest_run and item.run_id == latest_run.id
),
None,
)
approvals = getattr(self.repository, "approval_decisions", [])
latest_approval = sorted(
[item for item in approvals if item.candidate_id == candidate_id],
key=lambda item: item.decided_at,
reverse=True,
)[0] if any(item.candidate_id == candidate_id for item in approvals) else None
latest_approval = (
sorted(
[item for item in approvals if item.candidate_id == candidate_id],
key=lambda item: item.decided_at,
reverse=True,
)[0]
if any(item.candidate_id == candidate_id for item in approvals)
else None
)
publications = getattr(self.repository, "publication_records", [])
latest_publication = sorted(
[item for item in publications if item.candidate_id == candidate_id],
key=lambda item: item.published_at,
reverse=True,
)[0] if any(item.candidate_id == candidate_id for item in publications) else None
latest_publication = (
sorted(
[item for item in publications if item.candidate_id == candidate_id],
key=lambda item: item.published_at,
reverse=True,
)[0]
if any(item.candidate_id == candidate_id for item in publications)
else None
)
policy = self.repository.get_active_policy()
registry = self.repository.get_registry(policy.internal_source_registry_ref) if policy else None
registry = (
self.repository.get_registry(policy.internal_source_registry_ref)
if policy
else None
)
return {
"candidate": candidate,
"manifest": latest_manifest,
@@ -139,6 +187,8 @@ class TuiFacadeAdapter:
"policy": policy,
"registry": registry,
}
# [/DEF:TuiFacadeAdapter:Class]
@@ -166,11 +216,11 @@ class CleanReleaseTUI:
curses.start_color()
curses.use_default_colors()
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) # Header/Footer
curses.init_pair(2, curses.COLOR_GREEN, -1) # PASS
curses.init_pair(3, curses.COLOR_RED, -1) # FAIL/BLOCKED
curses.init_pair(4, curses.COLOR_YELLOW, -1) # RUNNING
curses.init_pair(5, curses.COLOR_CYAN, -1) # Text
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLUE) # Header/Footer
curses.init_pair(2, curses.COLOR_GREEN, -1) # PASS
curses.init_pair(3, curses.COLOR_RED, -1) # FAIL/BLOCKED
curses.init_pair(4, curses.COLOR_YELLOW, -1) # RUNNING
curses.init_pair(5, curses.COLOR_CYAN, -1) # Text
def _build_repository(self, mode: str) -> CleanReleaseRepository:
repo = CleanReleaseRepository()
@@ -317,7 +367,9 @@ class CleanReleaseTUI:
"prohibited_artifact_categories",
["test-data", "demo", "load-test"],
),
required_system_categories=payload.get("required_system_categories", ["core"]),
required_system_categories=payload.get(
"required_system_categories", ["core"]
),
effective_from=now,
)
)
@@ -354,18 +406,21 @@ class CleanReleaseTUI:
self.stdscr.addstr(4, 3, "Checks:")
check_defs = [
(CheckStageName.DATA_PURITY, "Data Purity (no test/demo payloads)"),
(CheckStageName.INTERNAL_SOURCES_ONLY, "Internal Sources Only (company servers)"),
(
CheckStageName.INTERNAL_SOURCES_ONLY,
"Internal Sources Only (company servers)",
),
(CheckStageName.NO_EXTERNAL_ENDPOINTS, "No External Internet Endpoints"),
(CheckStageName.MANIFEST_CONSISTENCY, "Release Manifest Consistency"),
]
row = 5
drawn_checks = {c["stage"]: c for c in self.checks_progress}
for stage, desc in check_defs:
status_text = " "
color = curses.color_pair(5)
if stage in drawn_checks:
c = drawn_checks[stage]
if c["status"] == "RUNNING":
@@ -377,7 +432,7 @@ class CleanReleaseTUI:
elif c["status"] == CheckStageStatus.FAIL:
status_text = "FAIL"
color = curses.color_pair(3)
self.stdscr.addstr(row, 4, f"[{status_text:^4}] {desc}")
if status_text != " ":
self.stdscr.addstr(row, 50, f"{status_text:>10}", color | curses.A_BOLD)
@@ -396,12 +451,18 @@ class CleanReleaseTUI:
def draw_status(self):
color = curses.color_pair(5)
if self.status == CheckFinalStatus.COMPLIANT: color = curses.color_pair(2)
elif self.status == CheckFinalStatus.BLOCKED: color = curses.color_pair(3)
stat_str = str(self.status.value if hasattr(self.status, "value") else self.status)
self.stdscr.addstr(18, 3, f"FINAL STATUS: {stat_str.upper()}", color | curses.A_BOLD)
if self.status == CheckFinalStatus.COMPLIANT:
color = curses.color_pair(2)
elif self.status == CheckFinalStatus.BLOCKED:
color = curses.color_pair(3)
stat_str = str(
self.status.value if hasattr(self.status, "value") else self.status
)
self.stdscr.addstr(
18, 3, f"FINAL STATUS: {stat_str.upper()}", color | curses.A_BOLD
)
if self.report_id:
self.stdscr.addstr(19, 3, f"Report ID: {self.report_id}")
@@ -413,22 +474,36 @@ class CleanReleaseTUI:
self.stdscr.addstr(20, 32, f"Publication: {publication.status}")
if self.violations_list:
self.stdscr.addstr(21, 3, f"Violations Details ({len(self.violations_list)} total):", curses.color_pair(3) | curses.A_BOLD)
self.stdscr.addstr(
21,
3,
f"Violations Details ({len(self.violations_list)} total):",
curses.color_pair(3) | curses.A_BOLD,
)
row = 22
for i, v in enumerate(self.violations_list[:5]):
v_cat = str(getattr(v, "code", "VIOLATION"))
msg = str(getattr(v, "message", "Violation detected"))
location = str(
getattr(v, "artifact_path", "")
or getattr(getattr(v, "evidence_json", {}), "get", lambda *_: "")("location", "")
or getattr(getattr(v, "evidence_json", {}), "get", lambda *_: "")(
"location", ""
)
)
msg_text = f"[{v_cat}] {msg} (Loc: {location})"
self.stdscr.addstr(row + i, 5, msg_text[:70], curses.color_pair(3))
if self.last_error:
self.stdscr.addstr(27, 3, f"Error: {self.last_error}"[:100], curses.color_pair(3) | curses.A_BOLD)
self.stdscr.addstr(
27,
3,
f"Error: {self.last_error}"[:100],
curses.color_pair(3) | curses.A_BOLD,
)
def draw_footer(self, max_y: int, max_x: int):
footer_text = " F5 Run F6 Manifest F7 Refresh F8 Approve F9 Publish F10 Exit ".center(max_x)
footer_text = " F5 Run F6 Manifest F7 Refresh F8 Approve F9 Publish F10 Exit ".center(
max_x
)
self.stdscr.attron(curses.color_pair(1))
self.stdscr.addstr(max_y - 1, 0, footer_text[:max_x])
self.stdscr.attroff(curses.color_pair(1))
@@ -446,7 +521,9 @@ class CleanReleaseTUI:
self.refresh_screen()
try:
result = self.facade.run_compliance(candidate_id=self.candidate_id, actor="operator")
result = self.facade.run_compliance(
candidate_id=self.candidate_id, actor="operator"
)
except Exception as exc: # noqa: BLE001
self.status = CheckFinalStatus.FAILED
self.last_error = str(exc)
@@ -456,7 +533,9 @@ class CleanReleaseTUI:
self.checks_progress = [
{
"stage": stage.stage_name,
"status": CheckStageStatus.PASS if str(stage.decision).upper() == "PASSED" else CheckStageStatus.FAIL,
"status": CheckStageStatus.PASS
if str(stage.decision).upper() == "PASSED"
else CheckStageStatus.FAIL,
}
for stage in result.stage_runs
]
@@ -472,11 +551,14 @@ class CleanReleaseTUI:
self.status = CheckFinalStatus.FAILED
self.refresh_overview()
self.refresh_screen()
# [/DEF:run_checks:Function]
def build_manifest(self):
try:
manifest = self.facade.build_manifest(candidate_id=self.candidate_id, actor="operator")
manifest = self.facade.build_manifest(
candidate_id=self.candidate_id, actor="operator"
)
self.status = "READY"
self.report_id = None
self.violations_list = []
@@ -570,11 +652,13 @@ class CleanReleaseTUI:
self.approve_latest()
elif char == curses.KEY_F9:
self.publish_latest()
# [/DEF:CleanReleaseTUI:Class]
def tui_main(stdscr: curses.window):
curses.curs_set(0) # Hide cursor
curses.curs_set(0) # Hide cursor
app = CleanReleaseTUI(stdscr)
app.loop()
@@ -597,4 +681,4 @@ def main() -> int:
if __name__ == "__main__":
sys.exit(main())
# [/DEF:backend.src.scripts.clean_release_tui:Module]
# [/DEF:CleanReleaseTuiScript:Module]

19
backend/src/scripts/create_admin.py
View File

@@ -1,12 +1,12 @@
# [DEF:backend.src.scripts.create_admin:Module]
# [DEF:CreateAdminScript:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: admin, setup, user, auth, cli
# @PURPOSE: CLI tool for creating the initial admin user.
# @LAYER: Scripts
# @RELATION: USES -> backend.src.core.auth.security
# @RELATION: USES -> backend.src.core.database
# @RELATION: USES -> backend.src.models.auth
# @RELATION: USES -> [AuthSecurityModule]
# @RELATION: USES -> [DatabaseModule]
# @RELATION: USES -> [AuthModels]
#
# @INVARIANT: Admin user must have the "Admin" role.
@@ -24,6 +24,7 @@ from src.models.auth import User, Role
from src.core.logger import logger, belief_scope
# [/SECTION]
# [DEF:create_admin:Function]
# @PURPOSE: Creates an admin user and necessary roles/permissions.
# @PRE: username and password provided via CLI.
@@ -36,7 +37,9 @@ def create_admin(username, password, email=None):
with belief_scope("create_admin"):
db = AuthSessionLocal()
try:
normalized_email = email.strip() if isinstance(email, str) and email.strip() else None
normalized_email = (
email.strip() if isinstance(email, str) and email.strip() else None
)
# 1. Ensure Admin role exists
admin_role = db.query(Role).filter(Role.name == "Admin").first()
@@ -60,7 +63,7 @@ def create_admin(username, password, email=None):
email=normalized_email,
password_hash=get_password_hash(password),
auth_source="LOCAL",
is_active=True
is_active=True,
)
new_user.roles.append(admin_role)
db.add(new_user)
@@ -74,6 +77,8 @@ def create_admin(username, password, email=None):
raise
finally:
db.close()
# [/DEF:create_admin:Function]
if __name__ == "__main__":
@@ -91,4 +96,4 @@ if __name__ == "__main__":
except Exception:
sys.exit(1)
# [/DEF:backend.src.scripts.create_admin:Module]
# [/DEF:CreateAdminScript:Module]

8
backend/src/scripts/dataset_dashboard_analysis.json
View File

@@ -10,7 +10,7 @@
},
"changed_by_name": "Superset Admin",
"changed_on": "2026-02-24T19:24:01.850617",
"changed_on_delta_humanized": "20 days ago",
"changed_on_delta_humanized": "29 days ago",
"charts": [
"TA-0001-001 test_chart"
],
@@ -19,7 +19,7 @@
"id": 1,
"last_name": "Admin"
},
"created_on_delta_humanized": "26 days ago",
"created_on_delta_humanized": "a month ago",
"css": null,
"dashboard_title": "TA-0001 Test dashboard",
"id": 13,
@@ -54,7 +54,7 @@
"last_name": "Admin"
},
"changed_on": "2026-02-18T14:56:04.863722",
"changed_on_humanized": "26 days ago",
"changed_on_humanized": "a month ago",
"column_formats": {},
"columns": [
{
@@ -424,7 +424,7 @@
"last_name": "Admin"
},
"created_on": "2026-02-18T14:56:04.317950",
"created_on_humanized": "26 days ago",
"created_on_humanized": "a month ago",
"database": {
"allow_multi_catalog": false,
"backend": "postgresql",

65
backend/src/scripts/migrate_sqlite_to_postgres.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.scripts.migrate_sqlite_to_postgres:Module]
# [DEF:MigrateSqliteToPostgresScript:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: migration, sqlite, postgresql, config, task_logs, task_records
@@ -30,7 +30,10 @@ from src.core.logger import belief_scope, logger
# [DEF:Constants:Section]
DEFAULT_TARGET_URL = os.getenv(
"DATABASE_URL",
os.getenv("POSTGRES_URL", "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools"),
os.getenv(
"POSTGRES_URL",
"postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools",
),
)
# [/DEF:Constants:Section]
@@ -56,6 +59,8 @@ def _json_load_if_needed(value: Any) -> Any:
except json.JSONDecodeError:
return value
return value
# [/DEF:_json_load_if_needed:Function]
@@ -75,6 +80,8 @@ def _find_legacy_config_path(explicit_path: Optional[str]) -> Optional[Path]:
if candidate.exists():
return candidate
return None
# [/DEF:_find_legacy_config_path:Function]
@@ -85,6 +92,8 @@ def _connect_sqlite(path: Path) -> sqlite3.Connection:
conn = sqlite3.connect(str(path))
conn.row_factory = sqlite3.Row
return conn
# [/DEF:_connect_sqlite:Function]
@@ -150,6 +159,8 @@ def _ensure_target_schema(engine) -> None:
with engine.begin() as conn:
for stmt in stmts:
conn.execute(text(stmt))
# [/DEF:_ensure_target_schema:Function]
@@ -158,7 +169,9 @@ def _ensure_target_schema(engine) -> None:
def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:
with belief_scope("_migrate_config"):
if legacy_config_path is None:
logger.info("[_migrate_config][Action] No legacy config.json found, skipping")
logger.info(
"[_migrate_config][Action] No legacy config.json found, skipping"
)
return 0
payload = json.loads(legacy_config_path.read_text(encoding="utf-8"))
@@ -174,8 +187,13 @@ def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:
),
{"payload": json.dumps(payload, ensure_ascii=True)},
)
logger.info("[_migrate_config][Coherence:OK] Config migrated from %s", legacy_config_path)
logger.info(
"[_migrate_config][Coherence:OK] Config migrated from %s",
legacy_config_path,
)
return 1
# [/DEF:_migrate_config:Function]
@@ -183,7 +201,12 @@ def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int:
# @PURPOSE: Migrates task_records and task_logs from SQLite into PostgreSQL.
def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str, int]:
with belief_scope("_migrate_tasks_and_logs"):
stats = {"task_records_total": 0, "task_records_inserted": 0, "task_logs_total": 0, "task_logs_inserted": 0}
stats = {
"task_records_total": 0,
"task_records_inserted": 0,
"task_logs_total": 0,
"task_logs_inserted": 0,
}
rows = sqlite_conn.execute(
"""
@@ -228,11 +251,17 @@ def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str
"environment_id": environment_id,
"started_at": row["started_at"],
"finished_at": row["finished_at"],
"logs": json.dumps(logs_obj, ensure_ascii=True) if logs_obj is not None else None,
"logs": json.dumps(logs_obj, ensure_ascii=True)
if logs_obj is not None
else None,
"error": row["error"],
"result": json.dumps(result_obj, ensure_ascii=True) if result_obj is not None else None,
"result": json.dumps(result_obj, ensure_ascii=True)
if result_obj is not None
else None,
"created_at": row["created_at"],
"params": json.dumps(params_obj, ensure_ascii=True) if params_obj is not None else None,
"params": json.dumps(params_obj, ensure_ascii=True)
if params_obj is not None
else None,
},
)
if res.rowcount and res.rowcount > 0:
@@ -292,14 +321,20 @@ def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str
stats["task_logs_total"],
)
return stats
# [/DEF:_migrate_tasks_and_logs:Function]
# [DEF:run_migration:Function]
# @PURPOSE: Orchestrates migration from SQLite/file to PostgreSQL.
def run_migration(sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]) -> Dict[str, int]:
def run_migration(
sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]
) -> Dict[str, int]:
with belief_scope("run_migration"):
logger.info("[run_migration][Entry] sqlite=%s target=%s", sqlite_path, target_url)
logger.info(
"[run_migration][Entry] sqlite=%s target=%s", sqlite_path, target_url
)
if not sqlite_path.exists():
raise FileNotFoundError(f"SQLite source not found: {sqlite_path}")
@@ -313,6 +348,8 @@ def run_migration(sqlite_path: Path, target_url: str, legacy_config_path: Option
return stats
finally:
sqlite_conn.close()
# [/DEF:run_migration:Function]
@@ -344,7 +381,11 @@ def main() -> int:
sqlite_path = Path(args.sqlite_path)
legacy_config_path = _find_legacy_config_path(args.config_path)
try:
stats = run_migration(sqlite_path=sqlite_path, target_url=args.target_url, legacy_config_path=legacy_config_path)
stats = run_migration(
sqlite_path=sqlite_path,
target_url=args.target_url,
legacy_config_path=legacy_config_path,
)
print("Migration completed.")
print(json.dumps(stats, indent=2))
return 0
@@ -358,4 +399,4 @@ if __name__ == "__main__":
raise SystemExit(main())
# [/DEF:main:Function]
# [/DEF:backend.src.scripts.migrate_sqlite_to_postgres:Module]
# [/DEF:MigrateSqliteToPostgresScript:Module]

157
backend/src/scripts/seed_superset_load_test.py
View File

@@ -1,11 +1,11 @@
# [DEF:backend.src.scripts.seed_superset_load_test:Module]
# [DEF:SeedSupersetLoadTestScript:Module]
#
# @COMPLEXITY: 3
# @SEMANTICS: superset, load-test, charts, dashboards, seed, stress
# @PURPOSE: Creates randomized load-test data in Superset by cloning chart configurations and creating dashboards in target environments.
# @LAYER: Scripts
# @RELATION: USES -> backend.src.core.config_manager.ConfigManager
# @RELATION: USES -> backend.src.core.superset_client.SupersetClient
# @RELATION: USES -> [ConfigManager]
# @RELATION: USES -> [SupersetClient]
# @INVARIANT: Created chart and dashboard names are globally unique for one script run.
# [SECTION: IMPORTS]
@@ -31,15 +31,42 @@ from src.core.superset_client import SupersetClient
# @PRE: Script is called from CLI.
# @POST: Returns validated argument namespace.
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Seed Superset with load-test charts and dashboards")
parser.add_argument("--envs", nargs="+", default=["ss1", "ss2"], help="Target environment IDs")
parser.add_argument("--charts", type=int, default=10000, help="Target number of charts to create")
parser.add_argument("--dashboards", type=int, default=500, help="Target number of dashboards to create")
parser.add_argument("--template-pool-size", type=int, default=200, help="How many source charts to sample as templates per env")
parser.add_argument("--seed", type=int, default=None, help="Optional RNG seed for reproducibility")
parser.add_argument("--max-errors", type=int, default=100, help="Stop early if errors exceed this threshold")
parser.add_argument("--dry-run", action="store_true", help="Do not write data, only validate setup")
parser = argparse.ArgumentParser(
description="Seed Superset with load-test charts and dashboards"
)
parser.add_argument(
"--envs", nargs="+", default=["ss1", "ss2"], help="Target environment IDs"
)
parser.add_argument(
"--charts", type=int, default=10000, help="Target number of charts to create"
)
parser.add_argument(
"--dashboards",
type=int,
default=500,
help="Target number of dashboards to create",
)
parser.add_argument(
"--template-pool-size",
type=int,
default=200,
help="How many source charts to sample as templates per env",
)
parser.add_argument(
"--seed", type=int, default=None, help="Optional RNG seed for reproducibility"
)
parser.add_argument(
"--max-errors",
type=int,
default=100,
help="Stop early if errors exceed this threshold",
)
parser.add_argument(
"--dry-run", action="store_true", help="Do not write data, only validate setup"
)
return parser.parse_args()
# [/DEF:_parse_args:Function]
@@ -52,6 +79,8 @@ def _extract_result_payload(payload: Dict) -> Dict:
if isinstance(result, dict):
return result
return payload
# [/DEF:_extract_result_payload:Function]
@@ -67,6 +96,8 @@ def _extract_created_id(payload: Dict) -> Optional[int]:
if isinstance(result, dict) and isinstance(result.get("id"), int):
return int(result["id"])
return None
# [/DEF:_extract_created_id:Function]
@@ -75,14 +106,38 @@ def _extract_created_id(payload: Dict) -> Optional[int]:
# @PRE: used_names is mutable set for collision tracking.
# @POST: Returns a unique string and stores it in used_names.
def _generate_unique_name(prefix: str, used_names: set[str], rng: random.Random) -> str:
adjectives = ["amber", "rapid", "frozen", "delta", "lunar", "vector", "cobalt", "silent", "neon", "solar"]
nouns = ["falcon", "matrix", "signal", "harbor", "stream", "vertex", "bridge", "orbit", "pulse", "forge"]
adjectives = [
"amber",
"rapid",
"frozen",
"delta",
"lunar",
"vector",
"cobalt",
"silent",
"neon",
"solar",
]
nouns = [
"falcon",
"matrix",
"signal",
"harbor",
"stream",
"vertex",
"bridge",
"orbit",
"pulse",
"forge",
]
while True:
token = uuid.uuid4().hex[:8]
candidate = f"{prefix}_{rng.choice(adjectives)}_{rng.choice(nouns)}_{rng.randint(100, 999)}_{token}"
if candidate not in used_names:
used_names.add(candidate)
return candidate
# [/DEF:_generate_unique_name:Function]
@@ -106,7 +161,9 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:
env = Environment(**row)
configured[env.id] = env
except Exception as exc:
logger.warning(f"[REFLECT] Failed loading environments from {config_path}: {exc}")
logger.warning(
f"[REFLECT] Failed loading environments from {config_path}: {exc}"
)
for env_id in env_ids:
env = configured.get(env_id)
@@ -115,6 +172,8 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:
resolved[env_id] = env
return resolved
# [/DEF:_resolve_target_envs:Function]
@@ -122,11 +181,21 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]:
# @PURPOSE: Builds a pool of source chart templates to clone in one environment.
# @PRE: Client is authenticated.
# @POST: Returns non-empty list of chart payload templates.
def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: random.Random) -> List[Dict]:
def _build_chart_template_pool(
client: SupersetClient, pool_size: int, rng: random.Random
) -> List[Dict]:
list_query = {
"page": 0,
"page_size": 1000,
"columns": ["id", "slice_name", "datasource_id", "datasource_type", "viz_type", "params", "query_context"],
"columns": [
"id",
"slice_name",
"datasource_id",
"datasource_type",
"viz_type",
"params",
"query_context",
],
}
rows = client.network.fetch_paginated_data(
endpoint="/chart/",
@@ -137,7 +206,11 @@ def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: rand
if not candidates:
raise RuntimeError("No source charts available for templating")
selected = candidates if len(candidates) <= pool_size else rng.sample(candidates, pool_size)
selected = (
candidates
if len(candidates) <= pool_size
else rng.sample(candidates, pool_size)
)
templates: List[Dict] = []
for row in selected:
@@ -146,7 +219,9 @@ def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: rand
detail = _extract_result_payload(detail_payload)
datasource_id = detail.get("datasource_id")
datasource_type = detail.get("datasource_type") or row.get("datasource_type") or "table"
datasource_type = (
detail.get("datasource_type") or row.get("datasource_type") or "table"
)
if datasource_id is None:
continue
@@ -172,6 +247,8 @@ def _build_chart_template_pool(client: SupersetClient, pool_size: int, rng: rand
raise RuntimeError("Could not build templates with datasource metadata")
return templates
# [/DEF:_build_chart_template_pool:Function]
@@ -195,23 +272,33 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
client = SupersetClient(env)
client.authenticate()
clients[env_id] = client
templates_by_env[env_id] = _build_chart_template_pool(client, args.template_pool_size, rng)
logger.info(f"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates")
templates_by_env[env_id] = _build_chart_template_pool(
client, args.template_pool_size, rng
)
logger.info(
f"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates"
)
errors = 0
env_ids = list(env_map.keys())
for idx in range(args.dashboards):
env_id = env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)
env_id = (
env_ids[idx % len(env_ids)] if idx < len(env_ids) else rng.choice(env_ids)
)
dashboard_title = _generate_unique_name("lt_dash", used_dashboard_names, rng)
if args.dry_run:
logger.info(f"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}")
logger.info(
f"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}"
)
continue
try:
payload = {"dashboard_title": dashboard_title, "published": False}
created = clients[env_id].network.request("POST", "/dashboard/", data=json.dumps(payload))
created = clients[env_id].network.request(
"POST", "/dashboard/", data=json.dumps(payload)
)
dashboard_id = _extract_created_id(created)
if dashboard_id is None:
raise RuntimeError(f"Dashboard create response missing id: {created}")
@@ -220,7 +307,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
errors += 1
logger.error(f"[EXPLORE] Failed creating dashboard in {env_id}: {exc}")
if errors >= args.max_errors:
raise RuntimeError(f"Stopping due to max errors reached ({errors})") from exc
raise RuntimeError(
f"Stopping due to max errors reached ({errors})"
) from exc
if args.dry_run:
return {
@@ -232,7 +321,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
for env_id in env_ids:
if not created_dashboards[env_id]:
raise RuntimeError(f"No dashboards created in environment {env_id}; cannot bind charts")
raise RuntimeError(
f"No dashboards created in environment {env_id}; cannot bind charts"
)
for index in range(args.charts):
env_id = rng.choice(env_ids)
@@ -255,7 +346,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
payload["query_context"] = template["query_context"]
try:
created = client.network.request("POST", "/chart/", data=json.dumps(payload))
created = client.network.request(
"POST", "/chart/", data=json.dumps(payload)
)
chart_id = _extract_created_id(created)
if chart_id is None:
raise RuntimeError(f"Chart create response missing id: {created}")
@@ -267,7 +360,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
errors += 1
logger.error(f"[EXPLORE] Failed creating chart in {env_id}: {exc}")
if errors >= args.max_errors:
raise RuntimeError(f"Stopping due to max errors reached ({errors})") from exc
raise RuntimeError(
f"Stopping due to max errors reached ({errors})"
) from exc
return {
"dry_run": False,
@@ -277,6 +372,8 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict:
"total_dashboards": sum(len(ids) for ids in created_dashboards.values()),
"total_charts": sum(len(ids) for ids in created_charts.values()),
}
# [/DEF:seed_superset_load_data:Function]
@@ -288,7 +385,9 @@ def main() -> None:
with belief_scope("seed_superset_load_test.main"):
args = _parse_args()
result = seed_superset_load_data(args)
logger.info(f"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}")
logger.info(
f"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}"
)
# [/DEF:main:Function]
@@ -297,4 +396,4 @@ def main() -> None:
if __name__ == "__main__":
main()
# [/DEF:backend.src.scripts.seed_superset_load_test:Module]
# [/DEF:SeedSupersetLoadTestScript:Module]