semantic
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -331,6 +331,7 @@ def test_parse_superset_link_dashboard_partial_recovery():
|
|||||||
)
|
)
|
||||||
fake_client = MagicMock()
|
fake_client = MagicMock()
|
||||||
fake_client.get_dashboard_detail.return_value = {
|
fake_client.get_dashboard_detail.return_value = {
|
||||||
|
"id": 10,
|
||||||
"datasets": [{"id": 42}, {"id": 77}],
|
"datasets": [{"id": 42}, {"id": 77}],
|
||||||
}
|
}
|
||||||
fake_client.get_dataset_detail.return_value = {
|
fake_client.get_dataset_detail.return_value = {
|
||||||
@@ -352,6 +353,123 @@ def test_parse_superset_link_dashboard_partial_recovery():
|
|||||||
# [/DEF:test_parse_superset_link_dashboard_partial_recovery:Function]
|
# [/DEF:test_parse_superset_link_dashboard_partial_recovery:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_parse_superset_link_dashboard_slug_recovery:Function]
|
||||||
|
# @PURPOSE: Verify dashboard slug links resolve through dashboard detail endpoints and recover dataset context.
|
||||||
|
def test_parse_superset_link_dashboard_slug_recovery():
|
||||||
|
env = Environment(
|
||||||
|
id="env-1",
|
||||||
|
name="DEV",
|
||||||
|
url="http://superset.local",
|
||||||
|
username="demo",
|
||||||
|
password="secret",
|
||||||
|
)
|
||||||
|
fake_client = MagicMock()
|
||||||
|
fake_client.get_dashboard_detail.return_value = {
|
||||||
|
"id": 15,
|
||||||
|
"datasets": [{"id": 42}],
|
||||||
|
}
|
||||||
|
fake_client.get_dataset_detail.return_value = {
|
||||||
|
"table_name": "sales",
|
||||||
|
"schema": "public",
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor = SupersetContextExtractor(environment=env, client=fake_client)
|
||||||
|
result = extractor.parse_superset_link(
|
||||||
|
"https://ss-dev.bebesh.ru/superset/dashboard/slack/?native_filters_key=8ZLV4M-UXOM"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.dataset_id == 42
|
||||||
|
assert result.dashboard_id == 15
|
||||||
|
assert result.dataset_ref == "public.sales"
|
||||||
|
assert result.partial_recovery is False
|
||||||
|
assert result.query_state["native_filters_key"] == "8ZLV4M-UXOM"
|
||||||
|
fake_client.get_dashboard_detail.assert_called_once_with("slack")
|
||||||
|
# [/DEF:test_parse_superset_link_dashboard_slug_recovery:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_parse_superset_link_dashboard_permalink_partial_recovery:Function]
|
||||||
|
# @PURPOSE: Verify dashboard permalink links no longer fail parsing and preserve permalink filter state for partial recovery.
|
||||||
|
def test_parse_superset_link_dashboard_permalink_partial_recovery():
|
||||||
|
env = Environment(
|
||||||
|
id="env-1",
|
||||||
|
name="DEV",
|
||||||
|
url="http://superset.local",
|
||||||
|
username="demo",
|
||||||
|
password="secret",
|
||||||
|
)
|
||||||
|
fake_client = MagicMock()
|
||||||
|
fake_client.get_dashboard_permalink_state.return_value = {
|
||||||
|
"state": {
|
||||||
|
"dataMask": {
|
||||||
|
"NATIVE_FILTER-1": {
|
||||||
|
"id": "country",
|
||||||
|
"filterState": {
|
||||||
|
"label": "Country",
|
||||||
|
"value": ["DE"],
|
||||||
|
},
|
||||||
|
"extraFormData": {
|
||||||
|
"filters": [{"col": "country", "op": "IN", "val": ["DE"]}],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
extractor = SupersetContextExtractor(environment=env, client=fake_client)
|
||||||
|
result = extractor.parse_superset_link(
|
||||||
|
"http://ss-dev.bebesh.ru/superset/dashboard/p/QabXy6wG30Z/"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.resource_type == "dashboard"
|
||||||
|
assert result.dataset_id is None
|
||||||
|
assert result.dashboard_id is None
|
||||||
|
assert result.dataset_ref == "dashboard_permalink:QabXy6wG30Z"
|
||||||
|
assert result.partial_recovery is True
|
||||||
|
assert "dashboard_permalink_dataset_binding_unresolved" in result.unresolved_references
|
||||||
|
assert result.imported_filters[0]["filter_name"] == "country"
|
||||||
|
assert result.imported_filters[0]["raw_value"] == ["DE"]
|
||||||
|
fake_client.get_dashboard_permalink_state.assert_called_once_with("QabXy6wG30Z")
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state:Function]
|
||||||
|
# @PURPOSE: Verify permalink state with nested dashboard id recovers dataset binding and keeps imported filters.
|
||||||
|
def test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state():
|
||||||
|
env = Environment(
|
||||||
|
id="env-1",
|
||||||
|
name="DEV",
|
||||||
|
url="http://superset.local",
|
||||||
|
username="demo",
|
||||||
|
password="secret",
|
||||||
|
)
|
||||||
|
fake_client = MagicMock()
|
||||||
|
fake_client.get_dashboard_permalink_state.return_value = {
|
||||||
|
"state": {
|
||||||
|
"form_data": {"dashboardId": 22},
|
||||||
|
"dataMask": {
|
||||||
|
"NATIVE_FILTER-1": {
|
||||||
|
"id": "country",
|
||||||
|
"filterState": {"label": "Country", "value": ["DE"]},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fake_client.get_dashboard_detail.return_value = {"id": 22, "datasets": [{"id": 42}]}
|
||||||
|
fake_client.get_dataset_detail.return_value = {"table_name": "sales", "schema": "public"}
|
||||||
|
|
||||||
|
extractor = SupersetContextExtractor(environment=env, client=fake_client)
|
||||||
|
result = extractor.parse_superset_link(
|
||||||
|
"http://ss-dev.bebesh.ru/superset/dashboard/p/QabXy6wG30Z/"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.dashboard_id == 22
|
||||||
|
assert result.dataset_id == 42
|
||||||
|
assert result.dataset_ref == "public.sales"
|
||||||
|
assert "dashboard_permalink_dataset_binding_unresolved" not in result.unresolved_references
|
||||||
|
assert result.imported_filters[0]["filter_name"] == "country"
|
||||||
|
# [/DEF:test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state:Function]
|
||||||
|
# [/DEF:test_parse_superset_link_dashboard_permalink_partial_recovery:Function]
|
||||||
|
|
||||||
|
|
||||||
# [DEF:test_resolve_from_dictionary_prefers_exact_match:Function]
|
# [DEF:test_resolve_from_dictionary_prefers_exact_match:Function]
|
||||||
# @PURPOSE: Verify trusted dictionary exact matches outrank fuzzy candidates and unresolved fields stay explicit.
|
# @PURPOSE: Verify trusted dictionary exact matches outrank fuzzy candidates and unresolved fields stay explicit.
|
||||||
def test_resolve_from_dictionary_prefers_exact_match():
|
def test_resolve_from_dictionary_prefers_exact_match():
|
||||||
@@ -400,6 +518,7 @@ def test_orchestrator_start_session_preserves_partial_recovery(dataset_review_ap
|
|||||||
|
|
||||||
repository.create_session.return_value = created_session
|
repository.create_session.return_value = created_session
|
||||||
repository.save_profile_and_findings.return_value = created_session
|
repository.save_profile_and_findings.return_value = created_session
|
||||||
|
repository.save_recovery_state.return_value = created_session
|
||||||
repository.db = MagicMock()
|
repository.db = MagicMock()
|
||||||
|
|
||||||
orchestrator = DatasetReviewOrchestrator(
|
orchestrator = DatasetReviewOrchestrator(
|
||||||
@@ -415,11 +534,23 @@ def test_orchestrator_start_session_preserves_partial_recovery(dataset_review_ap
|
|||||||
chart_id=None,
|
chart_id=None,
|
||||||
partial_recovery=True,
|
partial_recovery=True,
|
||||||
unresolved_references=["dashboard_dataset_binding_missing"],
|
unresolved_references=["dashboard_dataset_binding_missing"],
|
||||||
|
imported_filters=[],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
fake_extractor = MagicMock()
|
||||||
|
fake_extractor.parse_superset_link.return_value = parsed_context
|
||||||
|
fake_extractor.recover_imported_filters.return_value = []
|
||||||
|
fake_extractor.client.get_dataset_detail.return_value = {
|
||||||
|
"id": 42,
|
||||||
|
"sql": "",
|
||||||
|
"columns": [],
|
||||||
|
"metrics": [],
|
||||||
|
}
|
||||||
|
fake_extractor.discover_template_variables.return_value = []
|
||||||
|
|
||||||
with patch(
|
with patch(
|
||||||
"src.services.dataset_review.orchestrator.SupersetContextExtractor.parse_superset_link",
|
"src.services.dataset_review.orchestrator.SupersetContextExtractor",
|
||||||
return_value=parsed_context,
|
side_effect=[fake_extractor, fake_extractor],
|
||||||
):
|
):
|
||||||
result = orchestrator.start_session(
|
result = orchestrator.start_session(
|
||||||
StartSessionCommand(
|
StartSessionCommand(
|
||||||
@@ -438,6 +569,94 @@ def test_orchestrator_start_session_preserves_partial_recovery(dataset_review_ap
|
|||||||
# [/DEF:test_orchestrator_start_session_preserves_partial_recovery:Function]
|
# [/DEF:test_orchestrator_start_session_preserves_partial_recovery:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_orchestrator_start_session_bootstraps_recovery_state:Function]
|
||||||
|
# @PURPOSE: Verify session start persists recovered filters, template variables, and initial execution mappings for review workspace bootstrap.
|
||||||
|
def test_orchestrator_start_session_bootstraps_recovery_state(dataset_review_api_dependencies):
|
||||||
|
repository = MagicMock()
|
||||||
|
created_session = _make_session()
|
||||||
|
created_session.readiness_state = ReadinessState.RECOVERY_REQUIRED
|
||||||
|
created_session.current_phase = SessionPhase.RECOVERY
|
||||||
|
|
||||||
|
repository.create_session.return_value = created_session
|
||||||
|
repository.save_profile_and_findings.return_value = created_session
|
||||||
|
repository.save_recovery_state.return_value = created_session
|
||||||
|
repository.db = MagicMock()
|
||||||
|
|
||||||
|
orchestrator = DatasetReviewOrchestrator(
|
||||||
|
repository=repository,
|
||||||
|
config_manager=dataset_review_api_dependencies["config_manager"],
|
||||||
|
task_manager=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
parsed_context = SimpleNamespace(
|
||||||
|
dataset_ref="public.sales",
|
||||||
|
dataset_id=42,
|
||||||
|
dashboard_id=10,
|
||||||
|
chart_id=None,
|
||||||
|
partial_recovery=True,
|
||||||
|
unresolved_references=["dashboard_dataset_binding_missing"],
|
||||||
|
imported_filters=[{"filter_name": "country", "raw_value": ["DE"]}],
|
||||||
|
)
|
||||||
|
|
||||||
|
fake_extractor = MagicMock()
|
||||||
|
fake_extractor.parse_superset_link.return_value = parsed_context
|
||||||
|
fake_extractor.recover_imported_filters.return_value = [
|
||||||
|
{
|
||||||
|
"filter_name": "country",
|
||||||
|
"display_name": "Country",
|
||||||
|
"raw_value": ["DE"],
|
||||||
|
"normalized_value": ["DE"],
|
||||||
|
"source": "superset_url",
|
||||||
|
"confidence_state": "imported",
|
||||||
|
"requires_confirmation": False,
|
||||||
|
"recovery_status": "recovered",
|
||||||
|
"notes": "Recovered from permalink state",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
fake_extractor.client.get_dataset_detail.return_value = {
|
||||||
|
"id": 42,
|
||||||
|
"sql": "select * from sales where country in {{ filter_values('country') }}",
|
||||||
|
"columns": [],
|
||||||
|
"metrics": [],
|
||||||
|
}
|
||||||
|
fake_extractor.discover_template_variables.return_value = [
|
||||||
|
{
|
||||||
|
"variable_name": "country",
|
||||||
|
"expression_source": "{{ filter_values('country') }}",
|
||||||
|
"variable_kind": "native_filter",
|
||||||
|
"is_required": True,
|
||||||
|
"default_value": None,
|
||||||
|
"mapping_status": "unmapped",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"src.services.dataset_review.orchestrator.SupersetContextExtractor",
|
||||||
|
side_effect=[fake_extractor, fake_extractor],
|
||||||
|
):
|
||||||
|
result = orchestrator.start_session(
|
||||||
|
StartSessionCommand(
|
||||||
|
user=dataset_review_api_dependencies["user"],
|
||||||
|
environment_id="env-1",
|
||||||
|
source_kind="superset_link",
|
||||||
|
source_input="http://superset.local/dashboard/10",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.session.readiness_state == ReadinessState.RECOVERY_REQUIRED
|
||||||
|
repository.save_recovery_state.assert_called_once()
|
||||||
|
saved_filters = repository.save_recovery_state.call_args.args[2]
|
||||||
|
saved_variables = repository.save_recovery_state.call_args.args[3]
|
||||||
|
saved_mappings = repository.save_recovery_state.call_args.args[4]
|
||||||
|
assert len(saved_filters) == 1
|
||||||
|
assert saved_filters[0].filter_name == "country"
|
||||||
|
assert len(saved_variables) == 1
|
||||||
|
assert saved_variables[0].variable_name == "country"
|
||||||
|
assert len(saved_mappings) == 1
|
||||||
|
assert saved_mappings[0].raw_input_value == ["DE"]
|
||||||
|
# [/DEF:test_orchestrator_start_session_bootstraps_recovery_state:Function]
|
||||||
|
|
||||||
|
|
||||||
# [DEF:test_start_session_endpoint_returns_created_summary:Function]
|
# [DEF:test_start_session_endpoint_returns_created_summary:Function]
|
||||||
# @PURPOSE: Verify POST session lifecycle endpoint returns a persisted ownership-scoped summary.
|
# @PURPOSE: Verify POST session lifecycle endpoint returns a persisted ownership-scoped summary.
|
||||||
def test_start_session_endpoint_returns_created_summary(dataset_review_api_dependencies):
|
def test_start_session_endpoint_returns_created_summary(dataset_review_api_dependencies):
|
||||||
@@ -828,6 +1047,54 @@ def test_us3_mapping_patch_approval_preview_and_launch_endpoints(dataset_review_
|
|||||||
# [/DEF:test_us3_mapping_patch_approval_preview_and_launch_endpoints:Function]
|
# [/DEF:test_us3_mapping_patch_approval_preview_and_launch_endpoints:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift:Function]
|
||||||
|
# @PURPOSE: Preview endpoint should preserve API contract and surface generic upstream preview failures without fabricating dashboard-not-found semantics for non-dashboard 404s.
|
||||||
|
def test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift(
|
||||||
|
dataset_review_api_dependencies,
|
||||||
|
):
|
||||||
|
session = _make_us3_session()
|
||||||
|
repository = MagicMock()
|
||||||
|
repository.load_session_detail.return_value = session
|
||||||
|
repository.db = MagicMock()
|
||||||
|
repository.event_logger = MagicMock(spec=SessionEventLogger)
|
||||||
|
|
||||||
|
failed_preview = SimpleNamespace(
|
||||||
|
preview_id="preview-failed",
|
||||||
|
session_id="sess-1",
|
||||||
|
preview_status=PreviewStatus.FAILED,
|
||||||
|
compiled_sql=None,
|
||||||
|
preview_fingerprint="fingerprint-failed",
|
||||||
|
compiled_by="superset",
|
||||||
|
error_code="superset_preview_failed",
|
||||||
|
error_details="RuntimeError: [API_FAILURE] API resource not found at endpoint '/chart/data' | Context: {'status_code': 404, 'endpoint': '/chart/data', 'subtype': 'not_found'}",
|
||||||
|
compiled_at=None,
|
||||||
|
created_at=datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
orchestrator = MagicMock()
|
||||||
|
orchestrator.prepare_launch_preview.return_value = PreparePreviewResult(
|
||||||
|
session=session,
|
||||||
|
preview=failed_preview,
|
||||||
|
blocked_reasons=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
app.dependency_overrides[_get_repository] = lambda: repository
|
||||||
|
app.dependency_overrides[_get_orchestrator] = lambda: orchestrator
|
||||||
|
|
||||||
|
response = client.post("/api/dataset-orchestration/sessions/sess-1/preview")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
payload = response.json()
|
||||||
|
assert payload["preview_id"] == "preview-failed"
|
||||||
|
assert payload["preview_status"] == "failed"
|
||||||
|
assert payload["compiled_sql"] is None
|
||||||
|
assert payload["compiled_by"] == "superset"
|
||||||
|
assert payload["error_code"] == "superset_preview_failed"
|
||||||
|
assert "/chart/data" in payload["error_details"]
|
||||||
|
assert "API resource not found" in payload["error_details"]
|
||||||
|
assert "Dashboard not found" not in payload["error_details"]
|
||||||
|
# [/DEF:test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift:Function]
|
||||||
|
|
||||||
|
|
||||||
# [DEF:test_us3_launch_endpoint_requires_launch_permission:Function]
|
# [DEF:test_us3_launch_endpoint_requires_launch_permission:Function]
|
||||||
# @PURPOSE: Launch endpoint should enforce the contract RBAC permission instead of the generic session-manage permission.
|
# @PURPOSE: Launch endpoint should enforce the contract RBAC permission instead of the generic session-manage permission.
|
||||||
def test_us3_launch_endpoint_requires_launch_permission(dataset_review_api_dependencies):
|
def test_us3_launch_endpoint_requires_launch_permission(dataset_review_api_dependencies):
|
||||||
|
|||||||
@@ -5,8 +5,10 @@
|
|||||||
# @LAYER: Domain
|
# @LAYER: Domain
|
||||||
# @RELATION: VERIFIES -> ConfigManager
|
# @RELATION: VERIFIES -> ConfigManager
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from src.core.config_manager import ConfigManager
|
from src.core.config_manager import ConfigManager
|
||||||
from src.core.config_models import AppConfig, GlobalSettings
|
from src.core.config_models import AppConfig, Environment, GlobalSettings
|
||||||
|
|
||||||
|
|
||||||
# [DEF:test_get_payload_preserves_legacy_sections:Function]
|
# [DEF:test_get_payload_preserves_legacy_sections:Function]
|
||||||
@@ -48,6 +50,115 @@ def test_save_config_accepts_raw_payload_and_keeps_extras(monkeypatch):
|
|||||||
assert manager.raw_payload["notifications"]["telegram"]["bot_token"] == "secret"
|
assert manager.raw_payload["notifications"]["telegram"]["bot_token"] == "secret"
|
||||||
assert manager.config.settings.migration_sync_cron == "0 2 * * *"
|
assert manager.config.settings.migration_sync_cron == "0 2 * * *"
|
||||||
assert persisted["payload"]["notifications"]["telegram"]["bot_token"] == "secret"
|
assert persisted["payload"]["notifications"]["telegram"]["bot_token"] == "secret"
|
||||||
# [/DEF:test_save_config_accepts_raw_payload_and_keeps_extras:Function]
|
|
||||||
|
|
||||||
|
# [DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function]
|
||||||
|
# @PURPOSE: Ensure saving config mirrors typed environments into relational records required by FK-backed session persistence.
|
||||||
|
def test_save_config_syncs_environment_records_for_fk_backed_flows():
|
||||||
|
manager = ConfigManager.__new__(ConfigManager)
|
||||||
|
manager.raw_payload = {}
|
||||||
|
manager.config = AppConfig(environments=[], settings=GlobalSettings())
|
||||||
|
|
||||||
|
added_records = []
|
||||||
|
deleted_records = []
|
||||||
|
existing_record = SimpleNamespace(
|
||||||
|
id="legacy-env",
|
||||||
|
name="Legacy",
|
||||||
|
url="http://legacy.local",
|
||||||
|
credentials_id="legacy-user",
|
||||||
|
)
|
||||||
|
|
||||||
|
class _FakeQuery:
|
||||||
|
def all(self):
|
||||||
|
return [existing_record]
|
||||||
|
|
||||||
|
class _FakeSession:
|
||||||
|
def query(self, model):
|
||||||
|
return _FakeQuery()
|
||||||
|
|
||||||
|
def add(self, value):
|
||||||
|
added_records.append(value)
|
||||||
|
|
||||||
|
def delete(self, value):
|
||||||
|
deleted_records.append(value)
|
||||||
|
|
||||||
|
session = _FakeSession()
|
||||||
|
config = AppConfig(
|
||||||
|
environments=[
|
||||||
|
Environment(
|
||||||
|
id="dev",
|
||||||
|
name="DEV",
|
||||||
|
url="http://superset.local",
|
||||||
|
username="demo",
|
||||||
|
password="secret",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
settings=GlobalSettings(),
|
||||||
|
)
|
||||||
|
|
||||||
|
manager._sync_environment_records(session, config)
|
||||||
|
|
||||||
|
assert len(added_records) == 1
|
||||||
|
assert added_records[0].id == "dev"
|
||||||
|
assert added_records[0].name == "DEV"
|
||||||
|
assert added_records[0].url == "http://superset.local"
|
||||||
|
assert added_records[0].credentials_id == "demo"
|
||||||
|
assert deleted_records == [existing_record]
|
||||||
|
# [/DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function]
|
||||||
|
# @PURPOSE: Ensure loading an existing DB-backed config also mirrors environment rows required by FK-backed runtime flows.
|
||||||
|
def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypatch):
|
||||||
|
manager = ConfigManager.__new__(ConfigManager)
|
||||||
|
manager.config_path = None
|
||||||
|
manager.raw_payload = {}
|
||||||
|
manager.config = AppConfig(environments=[], settings=GlobalSettings())
|
||||||
|
|
||||||
|
sync_calls = []
|
||||||
|
closed = {"value": False}
|
||||||
|
committed = {"value": False}
|
||||||
|
|
||||||
|
class _FakeSession:
|
||||||
|
def commit(self):
|
||||||
|
committed["value"] = True
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
closed["value"] = True
|
||||||
|
|
||||||
|
fake_session = _FakeSession()
|
||||||
|
fake_record = SimpleNamespace(
|
||||||
|
id="global",
|
||||||
|
payload={
|
||||||
|
"environments": [
|
||||||
|
{
|
||||||
|
"id": "dev",
|
||||||
|
"name": "DEV",
|
||||||
|
"url": "http://superset.local",
|
||||||
|
"username": "demo",
|
||||||
|
"password": "secret",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"settings": GlobalSettings().model_dump(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr("src.core.config_manager.SessionLocal", lambda: fake_session)
|
||||||
|
monkeypatch.setattr(manager, "_get_record", lambda session: fake_record)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
manager,
|
||||||
|
"_sync_environment_records",
|
||||||
|
lambda session, config: sync_calls.append((session, config)),
|
||||||
|
)
|
||||||
|
|
||||||
|
config = manager._load_config()
|
||||||
|
|
||||||
|
assert config.environments[0].id == "dev"
|
||||||
|
assert len(sync_calls) == 1
|
||||||
|
assert sync_calls[0][0] is fake_session
|
||||||
|
assert sync_calls[0][1].environments[0].id == "dev"
|
||||||
|
assert committed["value"] is True
|
||||||
|
assert closed["value"] is True
|
||||||
|
# [/DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function]
|
||||||
|
|
||||||
# [/DEF:backend.src.core.__tests__.test_config_manager_compat:Module]
|
# [/DEF:backend.src.core.__tests__.test_config_manager_compat:Module]
|
||||||
|
|||||||
196
backend/src/core/__tests__/test_superset_preview_pipeline.py
Normal file
196
backend/src/core/__tests__/test_superset_preview_pipeline.py
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
# [DEF:SupersetPreviewPipelineTests:Module]
|
||||||
|
# @COMPLEXITY: 3
|
||||||
|
# @SEMANTICS: tests, superset, preview, chart_data, network, 404-mapping
|
||||||
|
# @PURPOSE: Verify explicit chart-data preview compilation and ensure non-dashboard 404 errors remain generic across sync and async clients.
|
||||||
|
# @LAYER: Domain
|
||||||
|
# @RELATION: [BINDS_TO] ->[SupersetClient]
|
||||||
|
# @RELATION: [BINDS_TO] ->[APIClient]
|
||||||
|
# @RELATION: [BINDS_TO] ->[AsyncAPIClient]
|
||||||
|
|
||||||
|
import json
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from src.core.config_models import Environment
|
||||||
|
from src.core.superset_client import SupersetClient
|
||||||
|
from src.core.utils.async_network import AsyncAPIClient
|
||||||
|
from src.core.utils.network import APIClient, DashboardNotFoundError, SupersetAPIError
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:_make_environment:Function]
|
||||||
|
def _make_environment() -> Environment:
|
||||||
|
return Environment(
|
||||||
|
id="env-1",
|
||||||
|
name="DEV",
|
||||||
|
url="http://superset.local",
|
||||||
|
username="demo",
|
||||||
|
password="secret",
|
||||||
|
)
|
||||||
|
# [/DEF:_make_environment:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:_make_requests_http_error:Function]
|
||||||
|
def _make_requests_http_error(status_code: int, url: str) -> requests.exceptions.HTTPError:
|
||||||
|
response = requests.Response()
|
||||||
|
response.status_code = status_code
|
||||||
|
response.url = url
|
||||||
|
response._content = b'{"message":"not found"}'
|
||||||
|
request = requests.Request("GET", url).prepare()
|
||||||
|
response.request = request
|
||||||
|
return requests.exceptions.HTTPError(response=response, request=request)
|
||||||
|
# [/DEF:_make_requests_http_error:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:_make_httpx_status_error:Function]
|
||||||
|
def _make_httpx_status_error(status_code: int, url: str) -> httpx.HTTPStatusError:
|
||||||
|
request = httpx.Request("GET", url)
|
||||||
|
response = httpx.Response(status_code=status_code, request=request, text='{"message":"not found"}')
|
||||||
|
return httpx.HTTPStatusError("upstream error", request=request, response=response)
|
||||||
|
# [/DEF:_make_httpx_status_error:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_compile_dataset_preview_uses_chart_data_and_result_query_sql:Function]
|
||||||
|
# @PURPOSE: Superset preview compilation should call the real chart-data endpoint and extract SQL from result[].query.
|
||||||
|
def test_compile_dataset_preview_uses_chart_data_and_result_query_sql():
|
||||||
|
client = SupersetClient(_make_environment())
|
||||||
|
client.get_dataset = MagicMock(
|
||||||
|
return_value={
|
||||||
|
"result": {
|
||||||
|
"id": 42,
|
||||||
|
"schema": "public",
|
||||||
|
"datasource": {"id": 42, "type": "table"},
|
||||||
|
"result_format": "json",
|
||||||
|
"result_type": "full",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
client.network = MagicMock()
|
||||||
|
client.network.request.return_value = {
|
||||||
|
"result": [
|
||||||
|
{
|
||||||
|
"query": "SELECT count(*) FROM public.sales WHERE country IN ('DE')",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
result = client.compile_dataset_preview(
|
||||||
|
dataset_id=42,
|
||||||
|
template_params={"country": "DE"},
|
||||||
|
effective_filters=[{"filter_name": "country", "effective_value": ["DE"]}],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["compiled_sql"] == "SELECT count(*) FROM public.sales WHERE country IN ('DE')"
|
||||||
|
client.network.request.assert_called_once()
|
||||||
|
request_call = client.network.request.call_args
|
||||||
|
assert request_call.kwargs["method"] == "POST"
|
||||||
|
assert request_call.kwargs["endpoint"] == "/chart/data"
|
||||||
|
assert request_call.kwargs["headers"] == {"Content-Type": "application/json"}
|
||||||
|
|
||||||
|
query_context = json.loads(request_call.kwargs["data"])
|
||||||
|
assert query_context["datasource"] == {"id": 42, "type": "table"}
|
||||||
|
assert query_context["queries"][0]["filters"] == [
|
||||||
|
{"col": "country", "op": "IN", "val": ["DE"]}
|
||||||
|
]
|
||||||
|
assert query_context["queries"][0]["url_params"] == {"country": "DE"}
|
||||||
|
|
||||||
|
assert result["query_context"]["datasource"] == {"id": 42, "type": "table"}
|
||||||
|
assert result["query_context"]["queries"][0]["filters"] == [
|
||||||
|
{"col": "country", "op": "IN", "val": ["DE"]}
|
||||||
|
]
|
||||||
|
# [/DEF:test_compile_dataset_preview_uses_chart_data_and_result_query_sql:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
|
||||||
|
# @PURPOSE: Sync network client should reserve dashboard-not-found translation for dashboard endpoints only.
|
||||||
|
def test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic():
|
||||||
|
client = APIClient(
|
||||||
|
config={
|
||||||
|
"base_url": "http://superset.local",
|
||||||
|
"auth": {"username": "demo", "password": "secret"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SupersetAPIError) as exc_info:
|
||||||
|
client._handle_http_error(
|
||||||
|
_make_requests_http_error(404, "http://superset.local/api/v1/chart/data"),
|
||||||
|
"/chart/data",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not isinstance(exc_info.value, DashboardNotFoundError)
|
||||||
|
assert "API resource not found at endpoint '/chart/data'" in str(exc_info.value)
|
||||||
|
# [/DEF:test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_sync_network_404_mapping_translates_dashboard_endpoints:Function]
|
||||||
|
# @PURPOSE: Sync network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors.
|
||||||
|
def test_sync_network_404_mapping_translates_dashboard_endpoints():
|
||||||
|
client = APIClient(
|
||||||
|
config={
|
||||||
|
"base_url": "http://superset.local",
|
||||||
|
"auth": {"username": "demo", "password": "secret"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(DashboardNotFoundError) as exc_info:
|
||||||
|
client._handle_http_error(
|
||||||
|
_make_requests_http_error(404, "http://superset.local/api/v1/dashboard/10"),
|
||||||
|
"/dashboard/10",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "Dashboard '/dashboard/10' Dashboard not found" in str(exc_info.value)
|
||||||
|
# [/DEF:test_sync_network_404_mapping_translates_dashboard_endpoints:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
|
||||||
|
# @PURPOSE: Async network client should reserve dashboard-not-found translation for dashboard endpoints only.
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic():
|
||||||
|
client = AsyncAPIClient(
|
||||||
|
config={
|
||||||
|
"base_url": "http://superset.local",
|
||||||
|
"auth": {"username": "demo", "password": "secret"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with pytest.raises(SupersetAPIError) as exc_info:
|
||||||
|
client._handle_http_error(
|
||||||
|
_make_httpx_status_error(404, "http://superset.local/api/v1/chart/data"),
|
||||||
|
"/chart/data",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not isinstance(exc_info.value, DashboardNotFoundError)
|
||||||
|
assert "API resource not found at endpoint '/chart/data'" in str(exc_info.value)
|
||||||
|
finally:
|
||||||
|
await client.aclose()
|
||||||
|
# [/DEF:test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [DEF:test_async_network_404_mapping_translates_dashboard_endpoints:Function]
|
||||||
|
# @PURPOSE: Async network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors.
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_network_404_mapping_translates_dashboard_endpoints():
|
||||||
|
client = AsyncAPIClient(
|
||||||
|
config={
|
||||||
|
"base_url": "http://superset.local",
|
||||||
|
"auth": {"username": "demo", "password": "secret"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with pytest.raises(DashboardNotFoundError) as exc_info:
|
||||||
|
client._handle_http_error(
|
||||||
|
_make_httpx_status_error(404, "http://superset.local/api/v1/dashboard/10"),
|
||||||
|
"/dashboard/10",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "Dashboard '/dashboard/10' Dashboard not found" in str(exc_info.value)
|
||||||
|
finally:
|
||||||
|
await client.aclose()
|
||||||
|
# [/DEF:test_async_network_404_mapping_translates_dashboard_endpoints:Function]
|
||||||
|
|
||||||
|
|
||||||
|
# [/DEF:SupersetPreviewPipelineTests:Module]
|
||||||
@@ -25,6 +25,7 @@ from sqlalchemy.orm import Session
|
|||||||
from .config_models import AppConfig, Environment, GlobalSettings
|
from .config_models import AppConfig, Environment, GlobalSettings
|
||||||
from .database import SessionLocal
|
from .database import SessionLocal
|
||||||
from ..models.config import AppConfigRecord
|
from ..models.config import AppConfigRecord
|
||||||
|
from ..models.mapping import Environment as EnvironmentRecord
|
||||||
from .logger import logger, configure_logger, belief_scope
|
from .logger import logger, configure_logger, belief_scope
|
||||||
|
|
||||||
|
|
||||||
@@ -146,6 +147,8 @@ class ConfigManager:
|
|||||||
"settings": self.raw_payload.get("settings", {}),
|
"settings": self.raw_payload.get("settings", {}),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
self._sync_environment_records(session, config)
|
||||||
|
session.commit()
|
||||||
logger.reason(
|
logger.reason(
|
||||||
"Database configuration validated successfully",
|
"Database configuration validated successfully",
|
||||||
extra={
|
extra={
|
||||||
@@ -202,6 +205,60 @@ class ConfigManager:
|
|||||||
session.close()
|
session.close()
|
||||||
# [/DEF:_load_config:Function]
|
# [/DEF:_load_config:Function]
|
||||||
|
|
||||||
|
# [DEF:_sync_environment_records:Function]
|
||||||
|
# @PURPOSE: Mirror configured environments into the relational environments table used by FK-backed domain models.
|
||||||
|
def _sync_environment_records(self, session: Session, config: AppConfig) -> None:
|
||||||
|
with belief_scope("ConfigManager._sync_environment_records"):
|
||||||
|
configured_envs = list(config.environments or [])
|
||||||
|
configured_ids = {
|
||||||
|
str(environment.id or "").strip()
|
||||||
|
for environment in configured_envs
|
||||||
|
if str(environment.id or "").strip()
|
||||||
|
}
|
||||||
|
|
||||||
|
persisted_records = session.query(EnvironmentRecord).all()
|
||||||
|
persisted_by_id = {str(record.id or "").strip(): record for record in persisted_records}
|
||||||
|
|
||||||
|
for environment in configured_envs:
|
||||||
|
normalized_id = str(environment.id or "").strip()
|
||||||
|
if not normalized_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
display_name = str(environment.name or normalized_id).strip() or normalized_id
|
||||||
|
normalized_url = str(environment.url or "").strip()
|
||||||
|
credentials_id = str(environment.username or "").strip() or normalized_id
|
||||||
|
|
||||||
|
record = persisted_by_id.get(normalized_id)
|
||||||
|
if record is None:
|
||||||
|
logger.reason(
|
||||||
|
"Creating relational environment record from typed config",
|
||||||
|
extra={"environment_id": normalized_id, "environment_name": display_name},
|
||||||
|
)
|
||||||
|
session.add(
|
||||||
|
EnvironmentRecord(
|
||||||
|
id=normalized_id,
|
||||||
|
name=display_name,
|
||||||
|
url=normalized_url,
|
||||||
|
credentials_id=credentials_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
record.name = display_name
|
||||||
|
record.url = normalized_url
|
||||||
|
record.credentials_id = credentials_id
|
||||||
|
|
||||||
|
for record in persisted_records:
|
||||||
|
normalized_id = str(record.id or "").strip()
|
||||||
|
if normalized_id and normalized_id not in configured_ids:
|
||||||
|
logger.reason(
|
||||||
|
"Removing stale relational environment record absent from typed config",
|
||||||
|
extra={"environment_id": normalized_id},
|
||||||
|
)
|
||||||
|
session.delete(record)
|
||||||
|
|
||||||
|
# [/DEF:_sync_environment_records:Function]
|
||||||
|
|
||||||
# [DEF:_save_config_to_db:Function]
|
# [DEF:_save_config_to_db:Function]
|
||||||
# @PURPOSE: Persist provided AppConfig into the global DB configuration record.
|
# @PURPOSE: Persist provided AppConfig into the global DB configuration record.
|
||||||
def _save_config_to_db(self, config: AppConfig, session: Optional[Session] = None) -> None:
|
def _save_config_to_db(self, config: AppConfig, session: Optional[Session] = None) -> None:
|
||||||
@@ -220,6 +277,8 @@ class ConfigManager:
|
|||||||
logger.reason("Updating existing global app config record", extra={"record_id": record.id})
|
logger.reason("Updating existing global app config record", extra={"record_id": record.id})
|
||||||
record.payload = payload
|
record.payload = payload
|
||||||
|
|
||||||
|
self._sync_environment_records(db, config)
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
logger.reason(
|
logger.reason(
|
||||||
"Configuration persisted to database",
|
"Configuration persisted to database",
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ from ..models import assistant as _assistant_models # noqa: F401
|
|||||||
from ..models import profile as _profile_models # noqa: F401
|
from ..models import profile as _profile_models # noqa: F401
|
||||||
from ..models import clean_release as _clean_release_models # noqa: F401
|
from ..models import clean_release as _clean_release_models # noqa: F401
|
||||||
from ..models import connection as _connection_models # noqa: F401
|
from ..models import connection as _connection_models # noqa: F401
|
||||||
|
from ..models import dataset_review as _dataset_review_models # noqa: F401
|
||||||
from .logger import belief_scope, logger
|
from .logger import belief_scope, logger
|
||||||
from .auth.config import auth_config
|
from .auth.config import auth_config
|
||||||
import os
|
import os
|
||||||
|
|||||||
@@ -1,11 +1,10 @@
|
|||||||
# [DEF:backend.src.core.superset_client:Module]
|
# [DEF:SupersetClientModule:Module]
|
||||||
#
|
#
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @SEMANTICS: superset, api, client, rest, http, dashboard, dataset, import, export
|
# @SEMANTICS: superset, api, client, rest, http, dashboard, dataset, import, export
|
||||||
# @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию.
|
# @PURPOSE: Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию.
|
||||||
# @LAYER: Core
|
# @LAYER: Core
|
||||||
# @RELATION: USES -> backend.src.core.utils.network.APIClient
|
# @RELATION: [DEPENDS_ON] ->[APIClient]
|
||||||
# @RELATION: USES -> backend.src.core.config_models.Environment
|
|
||||||
#
|
#
|
||||||
# @INVARIANT: All network operations must use the internal APIClient instance.
|
# @INVARIANT: All network operations must use the internal APIClient instance.
|
||||||
# @PUBLIC_API: SupersetClient
|
# @PUBLIC_API: SupersetClient
|
||||||
@@ -14,6 +13,7 @@
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import zipfile
|
import zipfile
|
||||||
|
from copy import deepcopy
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||||
from requests import Response
|
from requests import Response
|
||||||
@@ -24,18 +24,18 @@ from .utils.fileio import get_filename_from_headers
|
|||||||
from .config_models import Environment
|
from .config_models import Environment
|
||||||
# [/SECTION]
|
# [/SECTION]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient:Class]
|
# [DEF:SupersetClient:Class]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Класс-обёртка над Superset REST API, предоставляющий методы для работы с дашбордами и датасетами.
|
# @PURPOSE: Класс-обёртка над Superset REST API, предоставляющий методы для работы с дашбордами и датасетами.
|
||||||
# @RELATION: [DEPENDS_ON] ->[backend.src.core.utils.network.APIClient]
|
# @RELATION: [DEPENDS_ON] ->[APIClient]
|
||||||
# @RELATION: [DEPENDS_ON] ->[backend.src.core.config_models.Environment]
|
|
||||||
class SupersetClient:
|
class SupersetClient:
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.__init__:Function]
|
# [DEF:SupersetClient.__init__:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
|
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
|
||||||
# @PRE: `env` должен быть валидным объектом Environment.
|
# @PRE: `env` должен быть валидным объектом Environment.
|
||||||
# @POST: Атрибуты `env` и `network` созданы и готовы к работе.
|
# @POST: Атрибуты `env` и `network` созданы и готовы к работе.
|
||||||
# @DATA_CONTRACT: Input[Environment] -> self.network[APIClient]
|
# @DATA_CONTRACT: Input[Environment] -> self.network[APIClient]
|
||||||
|
# @RELATION: [DEPENDS_ON] ->[APIClient]
|
||||||
def __init__(self, env: Environment):
|
def __init__(self, env: Environment):
|
||||||
with belief_scope("__init__"):
|
with belief_scope("__init__"):
|
||||||
app_logger.info("[SupersetClient.__init__][Enter] Initializing SupersetClient for env %s.", env.name)
|
app_logger.info("[SupersetClient.__init__][Enter] Initializing SupersetClient for env %s.", env.name)
|
||||||
@@ -57,22 +57,22 @@ class SupersetClient:
|
|||||||
)
|
)
|
||||||
self.delete_before_reimport: bool = False
|
self.delete_before_reimport: bool = False
|
||||||
app_logger.info("[SupersetClient.__init__][Exit] SupersetClient initialized.")
|
app_logger.info("[SupersetClient.__init__][Exit] SupersetClient initialized.")
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.__init__:Function]
|
# [/DEF:SupersetClient.__init__:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.authenticate:Function]
|
# [DEF:SupersetClient.authenticate:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Authenticates the client using the configured credentials.
|
# @PURPOSE: Authenticates the client using the configured credentials.
|
||||||
# @PRE: self.network must be initialized with valid auth configuration.
|
# @PRE: self.network must be initialized with valid auth configuration.
|
||||||
# @POST: Client is authenticated and tokens are stored.
|
# @POST: Client is authenticated and tokens are stored.
|
||||||
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
|
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
|
||||||
# @RELATION: [CALLS] ->[self.network.authenticate]
|
# @RELATION: [CALLS] ->[APIClient.authenticate]
|
||||||
def authenticate(self) -> Dict[str, str]:
|
def authenticate(self) -> Dict[str, str]:
|
||||||
with belief_scope("SupersetClient.authenticate"):
|
with belief_scope("SupersetClient.authenticate"):
|
||||||
return self.network.authenticate()
|
return self.network.authenticate()
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.authenticate:Function]
|
# [/DEF:SupersetClient.authenticate:Function]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.headers:Function]
|
# [DEF:SupersetClient.headers:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
|
# @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
|
||||||
# @PRE: APIClient is initialized and authenticated.
|
# @PRE: APIClient is initialized and authenticated.
|
||||||
@@ -80,17 +80,17 @@ class SupersetClient:
|
|||||||
def headers(self) -> dict:
|
def headers(self) -> dict:
|
||||||
with belief_scope("headers"):
|
with belief_scope("headers"):
|
||||||
return self.network.headers
|
return self.network.headers
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.headers:Function]
|
# [/DEF:SupersetClient.headers:Function]
|
||||||
|
|
||||||
# [SECTION: DASHBOARD OPERATIONS]
|
# [SECTION: DASHBOARD OPERATIONS]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards:Function]
|
# [DEF:SupersetClient.get_dashboards:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.
|
# @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.
|
||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
# @POST: Returns a tuple with total count and list of dashboards.
|
# @POST: Returns a tuple with total count and list of dashboards.
|
||||||
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
||||||
# @RELATION: [CALLS] ->[self._fetch_all_pages]
|
# @RELATION: [CALLS] ->[SupersetClient._fetch_all_pages]
|
||||||
def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||||
with belief_scope("get_dashboards"):
|
with belief_scope("get_dashboards"):
|
||||||
app_logger.info("[get_dashboards][Enter] Fetching dashboards.")
|
app_logger.info("[get_dashboards][Enter] Fetching dashboards.")
|
||||||
@@ -116,15 +116,15 @@ class SupersetClient:
|
|||||||
total_count = len(paginated_data)
|
total_count = len(paginated_data)
|
||||||
app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count)
|
app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count)
|
||||||
return total_count, paginated_data
|
return total_count, paginated_data
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboards:Function]
|
# [/DEF:SupersetClient.get_dashboards:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_page:Function]
|
# [DEF:SupersetClient.get_dashboards_page:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.
|
# @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.
|
||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
# @POST: Returns total count and one page of dashboards.
|
# @POST: Returns total count and one page of dashboards.
|
||||||
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[APIClient.request]
|
||||||
def get_dashboards_page(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
def get_dashboards_page(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||||
with belief_scope("get_dashboards_page"):
|
with belief_scope("get_dashboards_page"):
|
||||||
validated_query = self._validate_query_params(query or {})
|
validated_query = self._validate_query_params(query or {})
|
||||||
@@ -153,15 +153,15 @@ class SupersetClient:
|
|||||||
result = response_json.get("result", [])
|
result = response_json.get("result", [])
|
||||||
total_count = response_json.get("count", len(result))
|
total_count = response_json.get("count", len(result))
|
||||||
return total_count, result
|
return total_count, result
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_page:Function]
|
# [/DEF:SupersetClient.get_dashboards_page:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_summary:Function]
|
# [DEF:SupersetClient.get_dashboards_summary:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches dashboard metadata optimized for the grid.
|
# @PURPOSE: Fetches dashboard metadata optimized for the grid.
|
||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
# @POST: Returns a list of dashboard metadata summaries.
|
# @POST: Returns a list of dashboard metadata summaries.
|
||||||
# @DATA_CONTRACT: None -> Output[List[Dict]]
|
# @DATA_CONTRACT: None -> Output[List[Dict]]
|
||||||
# @RELATION: [CALLS] ->[self.get_dashboards]
|
# @RELATION: [CALLS] ->[SupersetClient.get_dashboards]
|
||||||
def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:
|
def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:
|
||||||
with belief_scope("SupersetClient.get_dashboards_summary"):
|
with belief_scope("SupersetClient.get_dashboards_summary"):
|
||||||
# Rely on list endpoint default projection to stay compatible
|
# Rely on list endpoint default projection to stay compatible
|
||||||
@@ -238,15 +238,15 @@ class SupersetClient:
|
|||||||
f"sampled={min(len(result), max_debug_samples)})"
|
f"sampled={min(len(result), max_debug_samples)})"
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_summary:Function]
|
# [/DEF:SupersetClient.get_dashboards_summary:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_summary_page:Function]
|
# [DEF:SupersetClient.get_dashboards_summary_page:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.
|
# @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.
|
||||||
# @PRE: page >= 1 and page_size > 0.
|
# @PRE: page >= 1 and page_size > 0.
|
||||||
# @POST: Returns mapped summaries and total dashboard count.
|
# @POST: Returns mapped summaries and total dashboard count.
|
||||||
# @DATA_CONTRACT: Input[page: int, page_size: int] -> Output[Tuple[int, List[Dict]]]
|
# @DATA_CONTRACT: Input[page: int, page_size: int] -> Output[Tuple[int, List[Dict]]]
|
||||||
# @RELATION: [CALLS] ->[self.get_dashboards_page]
|
# @RELATION: [CALLS] ->[SupersetClient.get_dashboards_page]
|
||||||
def get_dashboards_summary_page(
|
def get_dashboards_summary_page(
|
||||||
self,
|
self,
|
||||||
page: int,
|
page: int,
|
||||||
@@ -313,7 +313,7 @@ class SupersetClient:
|
|||||||
return total_count, result
|
return total_count, result
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_summary_page:Function]
|
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboards_summary_page:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient._extract_owner_labels:Function]
|
# [DEF:SupersetClient._extract_owner_labels:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Normalize dashboard owners payload to stable display labels.
|
# @PURPOSE: Normalize dashboard owners payload to stable display labels.
|
||||||
# @PRE: owners payload can be scalar, object or list.
|
# @PRE: owners payload can be scalar, object or list.
|
||||||
@@ -339,9 +339,9 @@ class SupersetClient:
|
|||||||
if label and label not in normalized:
|
if label and label not in normalized:
|
||||||
normalized.append(label)
|
normalized.append(label)
|
||||||
return normalized
|
return normalized
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient._extract_owner_labels:Function]
|
# [/DEF:SupersetClient._extract_owner_labels:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient._extract_user_display:Function]
|
# [DEF:SupersetClient._extract_user_display:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Normalize user payload to a stable display name.
|
# @PURPOSE: Normalize user payload to a stable display name.
|
||||||
# @PRE: user payload can be string, dict or None.
|
# @PRE: user payload can be string, dict or None.
|
||||||
@@ -384,43 +384,59 @@ class SupersetClient:
|
|||||||
return normalized
|
return normalized
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient._sanitize_user_text:Function]
|
# [/DEF:backend.src.core.superset_client.SupersetClient._sanitize_user_text:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboard:Function]
|
# [DEF:SupersetClient.get_dashboard:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches a single dashboard by ID.
|
# @PURPOSE: Fetches a single dashboard by ID or slug.
|
||||||
# @PRE: Client is authenticated and dashboard_id exists.
|
# @PRE: Client is authenticated and dashboard_ref exists.
|
||||||
# @POST: Returns dashboard payload from Superset API.
|
# @POST: Returns dashboard payload from Superset API.
|
||||||
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict]
|
# @DATA_CONTRACT: Input[dashboard_ref: Union[int, str]] -> Output[Dict]
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[APIClient.request]
|
||||||
def get_dashboard(self, dashboard_id: int) -> Dict:
|
def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict:
|
||||||
with belief_scope("SupersetClient.get_dashboard", f"id={dashboard_id}"):
|
with belief_scope("SupersetClient.get_dashboard", f"ref={dashboard_ref}"):
|
||||||
response = self.network.request(method="GET", endpoint=f"/dashboard/{dashboard_id}")
|
response = self.network.request(method="GET", endpoint=f"/dashboard/{dashboard_ref}")
|
||||||
return cast(Dict, response)
|
return cast(Dict, response)
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboard:Function]
|
# [/DEF:SupersetClient.get_dashboard:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_chart:Function]
|
# [DEF:SupersetClient.get_dashboard_permalink_state:Function]
|
||||||
|
# @COMPLEXITY: 2
|
||||||
|
# @PURPOSE: Fetches stored dashboard permalink state by permalink key.
|
||||||
|
# @PRE: Client is authenticated and permalink key exists.
|
||||||
|
# @POST: Returns dashboard permalink state payload from Superset API.
|
||||||
|
# @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict]
|
||||||
|
# @RELATION: [CALLS] ->[APIClient.request]
|
||||||
|
def get_dashboard_permalink_state(self, permalink_key: str) -> Dict:
|
||||||
|
with belief_scope("SupersetClient.get_dashboard_permalink_state", f"key={permalink_key}"):
|
||||||
|
response = self.network.request(
|
||||||
|
method="GET",
|
||||||
|
endpoint=f"/dashboard/permalink/{permalink_key}"
|
||||||
|
)
|
||||||
|
return cast(Dict, response)
|
||||||
|
# [/DEF:SupersetClient.get_dashboard_permalink_state:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetClient.get_chart:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches a single chart by ID.
|
# @PURPOSE: Fetches a single chart by ID.
|
||||||
# @PRE: Client is authenticated and chart_id exists.
|
# @PRE: Client is authenticated and chart_id exists.
|
||||||
# @POST: Returns chart payload from Superset API.
|
# @POST: Returns chart payload from Superset API.
|
||||||
# @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict]
|
# @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict]
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[APIClient.request]
|
||||||
def get_chart(self, chart_id: int) -> Dict:
|
def get_chart(self, chart_id: int) -> Dict:
|
||||||
with belief_scope("SupersetClient.get_chart", f"id={chart_id}"):
|
with belief_scope("SupersetClient.get_chart", f"id={chart_id}"):
|
||||||
response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}")
|
response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}")
|
||||||
return cast(Dict, response)
|
return cast(Dict, response)
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_chart:Function]
|
# [/DEF:SupersetClient.get_chart:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_dashboard_detail:Function]
|
# [DEF:SupersetClient.get_dashboard_detail:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches detailed dashboard information including related charts and datasets.
|
# @PURPOSE: Fetches detailed dashboard information including related charts and datasets.
|
||||||
# @PRE: Client is authenticated and dashboard_id exists.
|
# @PRE: Client is authenticated and dashboard reference exists.
|
||||||
# @POST: Returns dashboard metadata with charts and datasets lists.
|
# @POST: Returns dashboard metadata with charts and datasets lists.
|
||||||
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict]
|
# @DATA_CONTRACT: Input[dashboard_ref: Union[int, str]] -> Output[Dict]
|
||||||
# @RELATION: [CALLS] ->[self.get_dashboard]
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_dashboard]
|
||||||
# @RELATION: [CALLS] ->[self.get_chart]
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_chart]
|
||||||
def get_dashboard_detail(self, dashboard_id: int) -> Dict:
|
def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:
|
||||||
with belief_scope("SupersetClient.get_dashboard_detail", f"id={dashboard_id}"):
|
with belief_scope("SupersetClient.get_dashboard_detail", f"ref={dashboard_ref}"):
|
||||||
dashboard_response = self.get_dashboard(dashboard_id)
|
dashboard_response = self.get_dashboard(dashboard_ref)
|
||||||
dashboard_data = dashboard_response.get("result", dashboard_response)
|
dashboard_data = dashboard_response.get("result", dashboard_response)
|
||||||
|
|
||||||
charts: List[Dict] = []
|
charts: List[Dict] = []
|
||||||
@@ -456,7 +472,7 @@ class SupersetClient:
|
|||||||
try:
|
try:
|
||||||
charts_response = self.network.request(
|
charts_response = self.network.request(
|
||||||
method="GET",
|
method="GET",
|
||||||
endpoint=f"/dashboard/{dashboard_id}/charts"
|
endpoint=f"/dashboard/{dashboard_ref}/charts"
|
||||||
)
|
)
|
||||||
charts_payload = charts_response.get("result", []) if isinstance(charts_response, dict) else []
|
charts_payload = charts_response.get("result", []) if isinstance(charts_response, dict) else []
|
||||||
for chart_obj in charts_payload:
|
for chart_obj in charts_payload:
|
||||||
@@ -486,7 +502,7 @@ class SupersetClient:
|
|||||||
try:
|
try:
|
||||||
datasets_response = self.network.request(
|
datasets_response = self.network.request(
|
||||||
method="GET",
|
method="GET",
|
||||||
endpoint=f"/dashboard/{dashboard_id}/datasets"
|
endpoint=f"/dashboard/{dashboard_ref}/datasets"
|
||||||
)
|
)
|
||||||
datasets_payload = datasets_response.get("result", []) if isinstance(datasets_response, dict) else []
|
datasets_payload = datasets_response.get("result", []) if isinstance(datasets_response, dict) else []
|
||||||
for dataset_obj in datasets_payload:
|
for dataset_obj in datasets_payload:
|
||||||
@@ -592,9 +608,10 @@ class SupersetClient:
|
|||||||
for dataset in datasets:
|
for dataset in datasets:
|
||||||
unique_datasets[dataset["id"]] = dataset
|
unique_datasets[dataset["id"]] = dataset
|
||||||
|
|
||||||
|
resolved_dashboard_id = dashboard_data.get("id", dashboard_ref)
|
||||||
return {
|
return {
|
||||||
"id": dashboard_data.get("id", dashboard_id),
|
"id": resolved_dashboard_id,
|
||||||
"title": dashboard_data.get("dashboard_title") or dashboard_data.get("title") or f"Dashboard {dashboard_id}",
|
"title": dashboard_data.get("dashboard_title") or dashboard_data.get("title") or f"Dashboard {resolved_dashboard_id}",
|
||||||
"slug": dashboard_data.get("slug"),
|
"slug": dashboard_data.get("slug"),
|
||||||
"url": dashboard_data.get("url"),
|
"url": dashboard_data.get("url"),
|
||||||
"description": dashboard_data.get("description") or "",
|
"description": dashboard_data.get("description") or "",
|
||||||
@@ -607,13 +624,13 @@ class SupersetClient:
|
|||||||
}
|
}
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboard_detail:Function]
|
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dashboard_detail:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_charts:Function]
|
# [DEF:SupersetClient.get_charts:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches all charts with pagination support.
|
# @PURPOSE: Fetches all charts with pagination support.
|
||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
# @POST: Returns total count and charts list.
|
# @POST: Returns total count and charts list.
|
||||||
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
||||||
# @RELATION: [CALLS] ->[self._fetch_all_pages]
|
# @RELATION: [CALLS] ->[SupersetClient._fetch_all_pages]
|
||||||
def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||||
with belief_scope("get_charts"):
|
with belief_scope("get_charts"):
|
||||||
validated_query = self._validate_query_params(query or {})
|
validated_query = self._validate_query_params(query or {})
|
||||||
@@ -625,9 +642,9 @@ class SupersetClient:
|
|||||||
pagination_options={"base_query": validated_query, "results_field": "result"},
|
pagination_options={"base_query": validated_query, "results_field": "result"},
|
||||||
)
|
)
|
||||||
return len(paginated_data), paginated_data
|
return len(paginated_data), paginated_data
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_charts:Function]
|
# [/DEF:SupersetClient.get_charts:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient._extract_chart_ids_from_layout:Function]
|
# [DEF:SupersetClient._extract_chart_ids_from_layout:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys.
|
# @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys.
|
||||||
# @PRE: payload can be dict/list/scalar.
|
# @PRE: payload can be dict/list/scalar.
|
||||||
@@ -667,7 +684,7 @@ class SupersetClient:
|
|||||||
# @POST: Returns ZIP content and filename.
|
# @POST: Returns ZIP content and filename.
|
||||||
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Tuple[bytes, str]]
|
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Tuple[bytes, str]]
|
||||||
# @SIDE_EFFECT: Performs network I/O to download archive.
|
# @SIDE_EFFECT: Performs network I/O to download archive.
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[backend.src.core.utils.network.APIClient.request]
|
||||||
def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:
|
def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:
|
||||||
with belief_scope("export_dashboard"):
|
with belief_scope("export_dashboard"):
|
||||||
app_logger.info("[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id)
|
app_logger.info("[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id)
|
||||||
@@ -692,8 +709,8 @@ class SupersetClient:
|
|||||||
# @POST: Dashboard is imported or re-imported after deletion.
|
# @POST: Dashboard is imported or re-imported after deletion.
|
||||||
# @DATA_CONTRACT: Input[file_name: Union[str, Path]] -> Output[Dict]
|
# @DATA_CONTRACT: Input[file_name: Union[str, Path]] -> Output[Dict]
|
||||||
# @SIDE_EFFECT: Performs network I/O to upload archive.
|
# @SIDE_EFFECT: Performs network I/O to upload archive.
|
||||||
# @RELATION: [CALLS] ->[self._do_import]
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient._do_import]
|
||||||
# @RELATION: [CALLS] ->[self.delete_dashboard]
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.delete_dashboard]
|
||||||
def import_dashboard(self, file_name: Union[str, Path], dash_id: Optional[int] = None, dash_slug: Optional[str] = None) -> Dict:
|
def import_dashboard(self, file_name: Union[str, Path], dash_id: Optional[int] = None, dash_slug: Optional[str] = None) -> Dict:
|
||||||
with belief_scope("import_dashboard"):
|
with belief_scope("import_dashboard"):
|
||||||
if file_name is None:
|
if file_name is None:
|
||||||
@@ -723,7 +740,7 @@ class SupersetClient:
|
|||||||
# @PRE: dashboard_id must exist.
|
# @PRE: dashboard_id must exist.
|
||||||
# @POST: Dashboard is removed from Superset.
|
# @POST: Dashboard is removed from Superset.
|
||||||
# @SIDE_EFFECT: Deletes resource from upstream Superset environment.
|
# @SIDE_EFFECT: Deletes resource from upstream Superset environment.
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[APIClient.request]
|
||||||
def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:
|
def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:
|
||||||
with belief_scope("delete_dashboard"):
|
with belief_scope("delete_dashboard"):
|
||||||
app_logger.info("[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id)
|
app_logger.info("[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id)
|
||||||
@@ -735,13 +752,13 @@ class SupersetClient:
|
|||||||
app_logger.warning("[delete_dashboard][Warning] Unexpected response while deleting %s: %s", dashboard_id, response)
|
app_logger.warning("[delete_dashboard][Warning] Unexpected response while deleting %s: %s", dashboard_id, response)
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.delete_dashboard:Function]
|
# [/DEF:backend.src.core.superset_client.SupersetClient.delete_dashboard:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_datasets:Function]
|
# [DEF:SupersetClient.get_datasets:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.
|
# @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.
|
||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
# @POST: Returns total count and list of datasets.
|
# @POST: Returns total count and list of datasets.
|
||||||
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
||||||
# @RELATION: [CALLS] ->[self._fetch_all_pages]
|
# @RELATION: [CALLS] ->[SupersetClient._fetch_all_pages]
|
||||||
def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||||
with belief_scope("get_datasets"):
|
with belief_scope("get_datasets"):
|
||||||
app_logger.info("[get_datasets][Enter] Fetching datasets.")
|
app_logger.info("[get_datasets][Enter] Fetching datasets.")
|
||||||
@@ -754,9 +771,9 @@ class SupersetClient:
|
|||||||
total_count = len(paginated_data)
|
total_count = len(paginated_data)
|
||||||
app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count)
|
app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count)
|
||||||
return total_count, paginated_data
|
return total_count, paginated_data
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_datasets:Function]
|
# [/DEF:SupersetClient.get_datasets:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_datasets_summary:Function]
|
# [DEF:SupersetClient.get_datasets_summary:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.
|
# @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.
|
||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
@@ -788,8 +805,8 @@ class SupersetClient:
|
|||||||
# @POST: Returns detailed dataset info with columns and linked dashboards.
|
# @POST: Returns detailed dataset info with columns and linked dashboards.
|
||||||
# @PARAM: dataset_id (int) - The dataset ID to fetch details for.
|
# @PARAM: dataset_id (int) - The dataset ID to fetch details for.
|
||||||
# @RETURN: Dict - Dataset details with columns and linked_dashboards.
|
# @RETURN: Dict - Dataset details with columns and linked_dashboards.
|
||||||
# @RELATION: CALLS -> self.get_dataset
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_dataset]
|
||||||
# @RELATION: CALLS -> self.network.request (for related_objects)
|
# @RELATION: [CALLS] ->[backend.src.core.utils.network.APIClient.request]
|
||||||
def get_dataset_detail(self, dataset_id: int) -> Dict:
|
def get_dataset_detail(self, dataset_id: int) -> Dict:
|
||||||
with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"):
|
with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"):
|
||||||
def as_bool(value, default=False):
|
def as_bool(value, default=False):
|
||||||
@@ -900,7 +917,7 @@ class SupersetClient:
|
|||||||
# @PRE: dataset_id must exist.
|
# @PRE: dataset_id must exist.
|
||||||
# @POST: Returns dataset details.
|
# @POST: Returns dataset details.
|
||||||
# @DATA_CONTRACT: Input[dataset_id: int] -> Output[Dict]
|
# @DATA_CONTRACT: Input[dataset_id: int] -> Output[Dict]
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[backend.src.core.utils.network.APIClient.request]
|
||||||
def get_dataset(self, dataset_id: int) -> Dict:
|
def get_dataset(self, dataset_id: int) -> Dict:
|
||||||
with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"):
|
with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"):
|
||||||
app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id)
|
app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id)
|
||||||
@@ -910,14 +927,196 @@ class SupersetClient:
|
|||||||
return response
|
return response
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dataset:Function]
|
# [/DEF:backend.src.core.superset_client.SupersetClient.get_dataset:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.update_dataset:Function]
|
# [DEF:SupersetClient.compile_dataset_preview:Function]
|
||||||
|
# @COMPLEXITY: 4
|
||||||
|
# @PURPOSE: Compile dataset preview SQL through the real Superset chart-data endpoint and return normalized SQL output.
|
||||||
|
# @PRE: dataset_id must be valid and template_params/effective_filters must represent the current preview session inputs.
|
||||||
|
# @POST: Returns normalized compiled SQL plus raw upstream response without guessing unsupported endpoints.
|
||||||
|
# @DATA_CONTRACT: Input[dataset_id:int, template_params:Dict, effective_filters:List[Dict]] -> Output[Dict[str, Any]]
|
||||||
|
# @RELATION: [CALLS] ->[SupersetClient.get_dataset]
|
||||||
|
# @RELATION: [CALLS] ->[SupersetClient.build_dataset_preview_query_context]
|
||||||
|
# @RELATION: [CALLS] ->[APIClient.request]
|
||||||
|
# @RELATION: [CALLS] ->[SupersetClient._extract_compiled_sql_from_chart_data_response]
|
||||||
|
# @SIDE_EFFECT: Performs upstream dataset lookup and chart-data network I/O against Superset.
|
||||||
|
def compile_dataset_preview(
|
||||||
|
self,
|
||||||
|
dataset_id: int,
|
||||||
|
template_params: Optional[Dict[str, Any]] = None,
|
||||||
|
effective_filters: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
with belief_scope("SupersetClient.compile_dataset_preview", f"id={dataset_id}"):
|
||||||
|
app_logger.reason(
|
||||||
|
"Compiling dataset preview via Superset chart-data endpoint",
|
||||||
|
extra={
|
||||||
|
"dataset_id": dataset_id,
|
||||||
|
"template_param_count": len(template_params or {}),
|
||||||
|
"filter_count": len(effective_filters or []),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
dataset_response = self.get_dataset(dataset_id)
|
||||||
|
dataset_record = dataset_response.get("result", dataset_response) if isinstance(dataset_response, dict) else {}
|
||||||
|
query_context = self.build_dataset_preview_query_context(
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
dataset_record=dataset_record,
|
||||||
|
template_params=template_params or {},
|
||||||
|
effective_filters=effective_filters or [],
|
||||||
|
)
|
||||||
|
response = self.network.request(
|
||||||
|
method="POST",
|
||||||
|
endpoint="/chart/data",
|
||||||
|
data=json.dumps(query_context),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
normalized = self._extract_compiled_sql_from_chart_data_response(response)
|
||||||
|
normalized["query_context"] = query_context
|
||||||
|
app_logger.reflect(
|
||||||
|
"Dataset preview compilation returned normalized SQL payload",
|
||||||
|
extra={
|
||||||
|
"dataset_id": dataset_id,
|
||||||
|
"compiled_sql_length": len(str(normalized.get("compiled_sql") or "")),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return normalized
|
||||||
|
# [/DEF:backend.src.core.superset_client.SupersetClient.compile_dataset_preview:Function]
|
||||||
|
|
||||||
|
# [DEF:backend.src.core.superset_client.SupersetClient.build_dataset_preview_query_context:Function]
|
||||||
|
# @COMPLEXITY: 4
|
||||||
|
# @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.
|
||||||
|
# @PRE: dataset_record should come from Superset dataset detail when possible.
|
||||||
|
# @POST: Returns an explicit chart-data payload based on current session inputs and dataset metadata.
|
||||||
|
# @DATA_CONTRACT: Input[dataset_id:int,dataset_record:Dict,template_params:Dict,effective_filters:List[Dict]] -> Output[Dict[str, Any]]
|
||||||
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient._normalize_effective_filters_for_query_context]
|
||||||
|
# @SIDE_EFFECT: Emits reasoning and reflection logs for deterministic preview payload construction.
|
||||||
|
def build_dataset_preview_query_context(
|
||||||
|
self,
|
||||||
|
dataset_id: int,
|
||||||
|
dataset_record: Dict[str, Any],
|
||||||
|
template_params: Dict[str, Any],
|
||||||
|
effective_filters: List[Dict[str, Any]],
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
with belief_scope("SupersetClient.build_dataset_preview_query_context", f"id={dataset_id}"):
|
||||||
|
normalized_template_params = deepcopy(template_params or {})
|
||||||
|
normalized_filters = self._normalize_effective_filters_for_query_context(effective_filters or [])
|
||||||
|
|
||||||
|
datasource_payload: Dict[str, Any] = {
|
||||||
|
"id": dataset_id,
|
||||||
|
"type": "table",
|
||||||
|
}
|
||||||
|
datasource = dataset_record.get("datasource")
|
||||||
|
if isinstance(datasource, dict):
|
||||||
|
datasource_id = datasource.get("id")
|
||||||
|
datasource_type = datasource.get("type")
|
||||||
|
if datasource_id is not None:
|
||||||
|
datasource_payload["id"] = datasource_id
|
||||||
|
if datasource_type:
|
||||||
|
datasource_payload["type"] = datasource_type
|
||||||
|
|
||||||
|
query_object: Dict[str, Any] = {
|
||||||
|
"filters": normalized_filters,
|
||||||
|
"extras": {"where": ""},
|
||||||
|
"columns": [],
|
||||||
|
"metrics": ["count"],
|
||||||
|
"orderby": [],
|
||||||
|
"annotation_layers": [],
|
||||||
|
"row_limit": 1000,
|
||||||
|
"series_limit": 0,
|
||||||
|
"url_params": normalized_template_params,
|
||||||
|
"custom_params": normalized_template_params,
|
||||||
|
}
|
||||||
|
|
||||||
|
schema = dataset_record.get("schema")
|
||||||
|
if schema:
|
||||||
|
query_object["schema"] = schema
|
||||||
|
|
||||||
|
time_range = dataset_record.get("default_time_range")
|
||||||
|
if time_range:
|
||||||
|
query_object["time_range"] = time_range
|
||||||
|
|
||||||
|
result_format = dataset_record.get("result_format") or "json"
|
||||||
|
result_type = dataset_record.get("result_type") or "full"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"datasource": datasource_payload,
|
||||||
|
"queries": [query_object],
|
||||||
|
"form_data": {
|
||||||
|
"datasource": f"{datasource_payload['id']}__{datasource_payload['type']}",
|
||||||
|
"viz_type": "table",
|
||||||
|
"slice_id": None,
|
||||||
|
"query_mode": "raw",
|
||||||
|
"url_params": normalized_template_params,
|
||||||
|
},
|
||||||
|
"result_format": result_format,
|
||||||
|
"result_type": result_type,
|
||||||
|
}
|
||||||
|
# [/DEF:backend.src.core.superset_client.SupersetClient.build_dataset_preview_query_context:Function]
|
||||||
|
|
||||||
|
# [DEF:backend.src.core.superset_client.SupersetClient._normalize_effective_filters_for_query_context:Function]
|
||||||
|
# @COMPLEXITY: 3
|
||||||
|
# @PURPOSE: Convert execution mappings into Superset chart-data filter objects.
|
||||||
|
# @PRE: effective_filters may contain mapping metadata and arbitrary scalar/list values.
|
||||||
|
# @POST: Returns only valid filter dictionaries suitable for the chart-data query payload.
|
||||||
|
def _normalize_effective_filters_for_query_context(
|
||||||
|
self,
|
||||||
|
effective_filters: List[Dict[str, Any]],
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
with belief_scope("SupersetClient._normalize_effective_filters_for_query_context"):
|
||||||
|
normalized_filters: List[Dict[str, Any]] = []
|
||||||
|
for item in effective_filters:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
column = str(item.get("variable_name") or item.get("filter_name") or "").strip()
|
||||||
|
if not column:
|
||||||
|
continue
|
||||||
|
value = item.get("effective_value")
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
operator = "IN" if isinstance(value, list) else "=="
|
||||||
|
normalized_filters.append(
|
||||||
|
{
|
||||||
|
"col": column,
|
||||||
|
"op": operator,
|
||||||
|
"val": value,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return normalized_filters
|
||||||
|
# [/DEF:backend.src.core.superset_client.SupersetClient._normalize_effective_filters_for_query_context:Function]
|
||||||
|
|
||||||
|
# [DEF:backend.src.core.superset_client.SupersetClient._extract_compiled_sql_from_chart_data_response:Function]
|
||||||
|
# @COMPLEXITY: 3
|
||||||
|
# @PURPOSE: Normalize compiled SQL from a chart-data response by reading result[].query fields first.
|
||||||
|
# @PRE: response must be the decoded response body from /api/v1/chart/data.
|
||||||
|
# @POST: Returns compiled SQL and raw response or raises SupersetAPIError when the endpoint does not expose query text.
|
||||||
|
def _extract_compiled_sql_from_chart_data_response(self, response: Any) -> Dict[str, Any]:
|
||||||
|
with belief_scope("SupersetClient._extract_compiled_sql_from_chart_data_response"):
|
||||||
|
if not isinstance(response, dict):
|
||||||
|
raise SupersetAPIError("Superset chart/data response was not a JSON object")
|
||||||
|
|
||||||
|
result_payload = response.get("result")
|
||||||
|
if not isinstance(result_payload, list):
|
||||||
|
raise SupersetAPIError("Superset chart/data response did not include a result list")
|
||||||
|
|
||||||
|
for item in result_payload:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
compiled_sql = str(item.get("query") or "").strip()
|
||||||
|
if compiled_sql:
|
||||||
|
return {
|
||||||
|
"compiled_sql": compiled_sql,
|
||||||
|
"raw_response": response,
|
||||||
|
}
|
||||||
|
|
||||||
|
raise SupersetAPIError("Superset chart/data response did not expose compiled SQL in result[].query")
|
||||||
|
# [/DEF:backend.src.core.superset_client.SupersetClient._extract_compiled_sql_from_chart_data_response:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetClient.update_dataset:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Обновляет данные датасета по его ID.
|
# @PURPOSE: Обновляет данные датасета по его ID.
|
||||||
# @PRE: dataset_id must exist.
|
# @PRE: dataset_id must exist.
|
||||||
# @POST: Dataset is updated in Superset.
|
# @POST: Dataset is updated in Superset.
|
||||||
# @DATA_CONTRACT: Input[dataset_id: int, data: Dict] -> Output[Dict]
|
# @DATA_CONTRACT: Input[dataset_id: int, data: Dict] -> Output[Dict]
|
||||||
# @SIDE_EFFECT: Modifies resource in upstream Superset environment.
|
# @SIDE_EFFECT: Modifies resource in upstream Superset environment.
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[APIClient.request]
|
||||||
def update_dataset(self, dataset_id: int, data: Dict) -> Dict:
|
def update_dataset(self, dataset_id: int, data: Dict) -> Dict:
|
||||||
with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"):
|
with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"):
|
||||||
app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id)
|
app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id)
|
||||||
@@ -930,15 +1129,15 @@ class SupersetClient:
|
|||||||
response = cast(Dict, response)
|
response = cast(Dict, response)
|
||||||
app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id)
|
app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id)
|
||||||
return response
|
return response
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.update_dataset:Function]
|
# [/DEF:SupersetClient.update_dataset:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_databases:Function]
|
# [DEF:SupersetClient.get_databases:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Получает полный список баз данных.
|
# @PURPOSE: Получает полный список баз данных.
|
||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
# @POST: Returns total count and list of databases.
|
# @POST: Returns total count and list of databases.
|
||||||
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
|
||||||
# @RELATION: [CALLS] ->[self._fetch_all_pages]
|
# @RELATION: [CALLS] ->[SupersetClient._fetch_all_pages]
|
||||||
def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
|
||||||
with belief_scope("get_databases"):
|
with belief_scope("get_databases"):
|
||||||
app_logger.info("[get_databases][Enter] Fetching databases.")
|
app_logger.info("[get_databases][Enter] Fetching databases.")
|
||||||
@@ -953,7 +1152,7 @@ class SupersetClient:
|
|||||||
total_count = len(paginated_data)
|
total_count = len(paginated_data)
|
||||||
app_logger.info("[get_databases][Exit] Found %d databases.", total_count)
|
app_logger.info("[get_databases][Exit] Found %d databases.", total_count)
|
||||||
return total_count, paginated_data
|
return total_count, paginated_data
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_databases:Function]
|
# [/DEF:SupersetClient.get_databases:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient.get_database:Function]
|
# [DEF:backend.src.core.superset_client.SupersetClient.get_database:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
@@ -961,7 +1160,7 @@ class SupersetClient:
|
|||||||
# @PRE: database_id must exist.
|
# @PRE: database_id must exist.
|
||||||
# @POST: Returns database details.
|
# @POST: Returns database details.
|
||||||
# @DATA_CONTRACT: Input[database_id: int] -> Output[Dict]
|
# @DATA_CONTRACT: Input[database_id: int] -> Output[Dict]
|
||||||
# @RELATION: [CALLS] ->[self.network.request]
|
# @RELATION: [CALLS] ->[backend.src.core.utils.network.APIClient.request]
|
||||||
def get_database(self, database_id: int) -> Dict:
|
def get_database(self, database_id: int) -> Dict:
|
||||||
with belief_scope("get_database"):
|
with belief_scope("get_database"):
|
||||||
app_logger.info("[get_database][Enter] Fetching database %s.", database_id)
|
app_logger.info("[get_database][Enter] Fetching database %s.", database_id)
|
||||||
@@ -977,7 +1176,7 @@ class SupersetClient:
|
|||||||
# @PRE: Client is authenticated.
|
# @PRE: Client is authenticated.
|
||||||
# @POST: Returns list of database summaries.
|
# @POST: Returns list of database summaries.
|
||||||
# @DATA_CONTRACT: None -> Output[List[Dict]]
|
# @DATA_CONTRACT: None -> Output[List[Dict]]
|
||||||
# @RELATION: [CALLS] ->[self.get_databases]
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_databases]
|
||||||
def get_databases_summary(self) -> List[Dict]:
|
def get_databases_summary(self) -> List[Dict]:
|
||||||
with belief_scope("SupersetClient.get_databases_summary"):
|
with belief_scope("SupersetClient.get_databases_summary"):
|
||||||
query = {
|
query = {
|
||||||
@@ -998,7 +1197,7 @@ class SupersetClient:
|
|||||||
# @PRE: db_uuid must be a valid UUID string.
|
# @PRE: db_uuid must be a valid UUID string.
|
||||||
# @POST: Returns database info or None.
|
# @POST: Returns database info or None.
|
||||||
# @DATA_CONTRACT: Input[db_uuid: str] -> Output[Optional[Dict]]
|
# @DATA_CONTRACT: Input[db_uuid: str] -> Output[Optional[Dict]]
|
||||||
# @RELATION: [CALLS] ->[self.get_databases]
|
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_databases]
|
||||||
def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]:
|
def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]:
|
||||||
with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"):
|
with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"):
|
||||||
query = {
|
query = {
|
||||||
@@ -1008,12 +1207,12 @@ class SupersetClient:
|
|||||||
return databases[0] if databases else None
|
return databases[0] if databases else None
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient.get_database_by_uuid:Function]
|
# [/DEF:backend.src.core.superset_client.SupersetClient.get_database_by_uuid:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient._resolve_target_id_for_delete:Function]
|
# [DEF:SupersetClient._resolve_target_id_for_delete:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Resolves a dashboard ID from either an ID or a slug.
|
# @PURPOSE: Resolves a dashboard ID from either an ID or a slug.
|
||||||
# @PRE: Either dash_id or dash_slug should be provided.
|
# @PRE: Either dash_id or dash_slug should be provided.
|
||||||
# @POST: Returns the resolved ID or None.
|
# @POST: Returns the resolved ID or None.
|
||||||
# @RELATION: [CALLS] ->[self.get_dashboards]
|
# @RELATION: [CALLS] ->[SupersetClient.get_dashboards]
|
||||||
def _resolve_target_id_for_delete(self, dash_id: Optional[int], dash_slug: Optional[str]) -> Optional[int]:
|
def _resolve_target_id_for_delete(self, dash_id: Optional[int], dash_slug: Optional[str]) -> Optional[int]:
|
||||||
with belief_scope("_resolve_target_id_for_delete"):
|
with belief_scope("_resolve_target_id_for_delete"):
|
||||||
if dash_id is not None:
|
if dash_id is not None:
|
||||||
@@ -1029,14 +1228,14 @@ class SupersetClient:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
app_logger.warning("[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s", dash_slug, e)
|
app_logger.warning("[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s", dash_slug, e)
|
||||||
return None
|
return None
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient._resolve_target_id_for_delete:Function]
|
# [/DEF:SupersetClient._resolve_target_id_for_delete:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient._do_import:Function]
|
# [DEF:SupersetClient._do_import:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Performs the actual multipart upload for import.
|
# @PURPOSE: Performs the actual multipart upload for import.
|
||||||
# @PRE: file_name must be a path to an existing ZIP file.
|
# @PRE: file_name must be a path to an existing ZIP file.
|
||||||
# @POST: Returns the API response from the upload.
|
# @POST: Returns the API response from the upload.
|
||||||
# @RELATION: [CALLS] ->[self.network.upload_file]
|
# @RELATION: [CALLS] ->[APIClient.upload_file]
|
||||||
def _do_import(self, file_name: Union[str, Path]) -> Dict:
|
def _do_import(self, file_name: Union[str, Path]) -> Dict:
|
||||||
with belief_scope("_do_import"):
|
with belief_scope("_do_import"):
|
||||||
app_logger.debug(f"[_do_import][State] Uploading file: {file_name}")
|
app_logger.debug(f"[_do_import][State] Uploading file: {file_name}")
|
||||||
@@ -1051,7 +1250,7 @@ class SupersetClient:
|
|||||||
extra_data={"overwrite": "true"},
|
extra_data={"overwrite": "true"},
|
||||||
timeout=self.env.timeout * 2,
|
timeout=self.env.timeout * 2,
|
||||||
)
|
)
|
||||||
# [/DEF:backend.src.core.superset_client.SupersetClient._do_import:Function]
|
# [/DEF:SupersetClient._do_import:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.superset_client.SupersetClient._validate_export_response:Function]
|
# [DEF:backend.src.core.superset_client.SupersetClient._validate_export_response:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
@@ -1101,7 +1300,7 @@ class SupersetClient:
|
|||||||
# @PURPOSE: Fetches the total number of items for a given endpoint.
|
# @PURPOSE: Fetches the total number of items for a given endpoint.
|
||||||
# @PRE: endpoint must be a valid Superset API path.
|
# @PRE: endpoint must be a valid Superset API path.
|
||||||
# @POST: Returns the total count as an integer.
|
# @POST: Returns the total count as an integer.
|
||||||
# @RELATION: [CALLS] ->[self.network.fetch_paginated_count]
|
# @RELATION: [CALLS] ->[backend.src.core.utils.network.APIClient.fetch_paginated_count]
|
||||||
def _fetch_total_object_count(self, endpoint: str) -> int:
|
def _fetch_total_object_count(self, endpoint: str) -> int:
|
||||||
with belief_scope("_fetch_total_object_count"):
|
with belief_scope("_fetch_total_object_count"):
|
||||||
return self.network.fetch_paginated_count(
|
return self.network.fetch_paginated_count(
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# [DEF:backend.src.core.utils.async_network:Module]
|
# [DEF:AsyncNetworkModule:Module]
|
||||||
#
|
#
|
||||||
# @COMPLEXITY: 5
|
# @COMPLEXITY: 5
|
||||||
# @SEMANTICS: network, httpx, async, superset, authentication, cache
|
# @SEMANTICS: network, httpx, async, superset, authentication, cache
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
|
# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
|
||||||
# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.
|
# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.
|
||||||
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
|
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
|
||||||
# @RELATION: DEPENDS_ON -> backend.src.core.utils.network.SupersetAuthCache
|
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
|
||||||
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
|
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
|
||||||
|
|
||||||
# [SECTION: IMPORTS]
|
# [SECTION: IMPORTS]
|
||||||
@@ -29,22 +29,24 @@ from .network import (
|
|||||||
# [/SECTION]
|
# [/SECTION]
|
||||||
|
|
||||||
|
|
||||||
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient:Class]
|
# [DEF:AsyncAPIClient:Class]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Async Superset API client backed by httpx.AsyncClient with shared auth cache.
|
# @PURPOSE: Async Superset API client backed by httpx.AsyncClient with shared auth cache.
|
||||||
# @RELATION: [DEPENDS_ON] ->[backend.src.core.utils.network.SupersetAuthCache]
|
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
|
||||||
# @RELATION: [CALLS] ->[backend.src.core.utils.network.SupersetAuthCache.get]
|
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
|
||||||
# @RELATION: [CALLS] ->[backend.src.core.utils.network.SupersetAuthCache.set]
|
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
|
||||||
class AsyncAPIClient:
|
class AsyncAPIClient:
|
||||||
DEFAULT_TIMEOUT = 30
|
DEFAULT_TIMEOUT = 30
|
||||||
_auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}
|
_auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}
|
||||||
|
|
||||||
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient.__init__:Function]
|
# [DEF:AsyncAPIClient.__init__:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Initialize async API client for one environment.
|
# @PURPOSE: Initialize async API client for one environment.
|
||||||
# @PRE: config contains base_url and auth payload.
|
# @PRE: config contains base_url and auth payload.
|
||||||
# @POST: Client is ready for async request/authentication flow.
|
# @POST: Client is ready for async request/authentication flow.
|
||||||
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
|
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
|
||||||
|
# @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]
|
||||||
|
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
|
||||||
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
|
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
|
||||||
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
|
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
|
||||||
self.api_base_url: str = f"{self.base_url}/api/v1"
|
self.api_base_url: str = f"{self.base_url}/api/v1"
|
||||||
@@ -63,9 +65,9 @@ class AsyncAPIClient:
|
|||||||
verify_ssl,
|
verify_ssl,
|
||||||
)
|
)
|
||||||
|
|
||||||
# [/DEF:__init__:Function]
|
# [/DEF:AsyncAPIClient.__init__:Function]
|
||||||
|
|
||||||
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient._normalize_base_url:Function]
|
# [DEF:AsyncAPIClient._normalize_base_url:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Normalize base URL for Superset API root construction.
|
# @PURPOSE: Normalize base URL for Superset API root construction.
|
||||||
# @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
|
# @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
|
||||||
@@ -74,9 +76,9 @@ class AsyncAPIClient:
|
|||||||
if normalized.lower().endswith("/api/v1"):
|
if normalized.lower().endswith("/api/v1"):
|
||||||
normalized = normalized[:-len("/api/v1")]
|
normalized = normalized[:-len("/api/v1")]
|
||||||
return normalized.rstrip("/")
|
return normalized.rstrip("/")
|
||||||
# [/DEF:_normalize_base_url:Function]
|
# [/DEF:AsyncAPIClient._normalize_base_url:Function]
|
||||||
|
|
||||||
# [DEF:_build_api_url:Function]
|
# [DEF:AsyncAPIClient._build_api_url:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Build full API URL from relative Superset endpoint.
|
# @PURPOSE: Build full API URL from relative Superset endpoint.
|
||||||
# @POST: Returns absolute URL for upstream request.
|
# @POST: Returns absolute URL for upstream request.
|
||||||
@@ -89,9 +91,9 @@ class AsyncAPIClient:
|
|||||||
if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1":
|
if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1":
|
||||||
return f"{self.base_url}{normalized_endpoint}"
|
return f"{self.base_url}{normalized_endpoint}"
|
||||||
return f"{self.api_base_url}{normalized_endpoint}"
|
return f"{self.api_base_url}{normalized_endpoint}"
|
||||||
# [/DEF:_build_api_url:Function]
|
# [/DEF:AsyncAPIClient._build_api_url:Function]
|
||||||
|
|
||||||
# [DEF:_get_auth_lock:Function]
|
# [DEF:AsyncAPIClient._get_auth_lock:Function]
|
||||||
# @COMPLEXITY: 1
|
# @COMPLEXITY: 1
|
||||||
# @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.
|
# @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.
|
||||||
# @POST: Returns stable asyncio.Lock instance.
|
# @POST: Returns stable asyncio.Lock instance.
|
||||||
@@ -103,14 +105,16 @@ class AsyncAPIClient:
|
|||||||
created_lock = asyncio.Lock()
|
created_lock = asyncio.Lock()
|
||||||
cls._auth_locks[cache_key] = created_lock
|
cls._auth_locks[cache_key] = created_lock
|
||||||
return created_lock
|
return created_lock
|
||||||
# [/DEF:_get_auth_lock:Function]
|
# [/DEF:AsyncAPIClient._get_auth_lock:Function]
|
||||||
|
|
||||||
# [DEF:authenticate:Function]
|
# [DEF:AsyncAPIClient.authenticate:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Authenticate against Superset and cache access/csrf tokens.
|
# @PURPOSE: Authenticate against Superset and cache access/csrf tokens.
|
||||||
# @POST: Client tokens are populated and reusable across requests.
|
# @POST: Client tokens are populated and reusable across requests.
|
||||||
# @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.
|
# @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.
|
||||||
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
|
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
|
||||||
|
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
|
||||||
|
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
|
||||||
async def authenticate(self) -> Dict[str, str]:
|
async def authenticate(self) -> Dict[str, str]:
|
||||||
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
|
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
|
||||||
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
|
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
|
||||||
@@ -163,13 +167,13 @@ class AsyncAPIClient:
|
|||||||
except (httpx.HTTPError, KeyError) as exc:
|
except (httpx.HTTPError, KeyError) as exc:
|
||||||
SupersetAuthCache.invalidate(self._auth_cache_key)
|
SupersetAuthCache.invalidate(self._auth_cache_key)
|
||||||
raise NetworkError(f"Network or parsing error during authentication: {exc}") from exc
|
raise NetworkError(f"Network or parsing error during authentication: {exc}") from exc
|
||||||
# [/DEF:authenticate:Function]
|
# [/DEF:AsyncAPIClient.authenticate:Function]
|
||||||
|
|
||||||
# [DEF:get_headers:Function]
|
# [DEF:AsyncAPIClient.get_headers:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Return authenticated Superset headers for async requests.
|
# @PURPOSE: Return authenticated Superset headers for async requests.
|
||||||
# @POST: Headers include Authorization and CSRF tokens.
|
# @POST: Headers include Authorization and CSRF tokens.
|
||||||
# @RELATION: CALLS -> self.authenticate
|
# @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]
|
||||||
async def get_headers(self) -> Dict[str, str]:
|
async def get_headers(self) -> Dict[str, str]:
|
||||||
if not self._authenticated:
|
if not self._authenticated:
|
||||||
await self.authenticate()
|
await self.authenticate()
|
||||||
@@ -179,16 +183,16 @@ class AsyncAPIClient:
|
|||||||
"Referer": self.base_url,
|
"Referer": self.base_url,
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
}
|
}
|
||||||
# [/DEF:get_headers:Function]
|
# [/DEF:AsyncAPIClient.get_headers:Function]
|
||||||
|
|
||||||
# [DEF:request:Function]
|
# [DEF:AsyncAPIClient.request:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Perform one authenticated async Superset API request.
|
# @PURPOSE: Perform one authenticated async Superset API request.
|
||||||
# @POST: Returns JSON payload or raw httpx.Response when raw_response=true.
|
# @POST: Returns JSON payload or raw httpx.Response when raw_response=true.
|
||||||
# @SIDE_EFFECT: Performs network I/O.
|
# @SIDE_EFFECT: Performs network I/O.
|
||||||
# @RELATION: [CALLS] ->[self.get_headers]
|
# @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]
|
||||||
# @RELATION: [CALLS] ->[self._handle_http_error]
|
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]
|
||||||
# @RELATION: [CALLS] ->[self._handle_network_error]
|
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]
|
||||||
async def request(
|
async def request(
|
||||||
self,
|
self,
|
||||||
method: str,
|
method: str,
|
||||||
@@ -216,9 +220,9 @@ class AsyncAPIClient:
|
|||||||
self._handle_http_error(exc, endpoint)
|
self._handle_http_error(exc, endpoint)
|
||||||
except httpx.HTTPError as exc:
|
except httpx.HTTPError as exc:
|
||||||
self._handle_network_error(exc, full_url)
|
self._handle_network_error(exc, full_url)
|
||||||
# [/DEF:request:Function]
|
# [/DEF:AsyncAPIClient.request:Function]
|
||||||
|
|
||||||
# [DEF:_handle_http_error:Function]
|
# [DEF:AsyncAPIClient._handle_http_error:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.
|
# @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.
|
||||||
# @POST: Raises domain-specific exception for caller flow control.
|
# @POST: Raises domain-specific exception for caller flow control.
|
||||||
@@ -229,15 +233,40 @@ class AsyncAPIClient:
|
|||||||
if status_code in [502, 503, 504]:
|
if status_code in [502, 503, 504]:
|
||||||
raise NetworkError(f"Environment unavailable (Status {status_code})", status_code=status_code) from exc
|
raise NetworkError(f"Environment unavailable (Status {status_code})", status_code=status_code) from exc
|
||||||
if status_code == 404:
|
if status_code == 404:
|
||||||
raise DashboardNotFoundError(endpoint) from exc
|
if self._is_dashboard_endpoint(endpoint):
|
||||||
|
raise DashboardNotFoundError(endpoint) from exc
|
||||||
|
raise SupersetAPIError(
|
||||||
|
f"API resource not found at endpoint '{endpoint}'",
|
||||||
|
status_code=status_code,
|
||||||
|
endpoint=endpoint,
|
||||||
|
subtype="not_found",
|
||||||
|
) from exc
|
||||||
if status_code == 403:
|
if status_code == 403:
|
||||||
raise PermissionDeniedError() from exc
|
raise PermissionDeniedError() from exc
|
||||||
if status_code == 401:
|
if status_code == 401:
|
||||||
raise AuthenticationError() from exc
|
raise AuthenticationError() from exc
|
||||||
raise SupersetAPIError(f"API Error {status_code}: {exc.response.text}") from exc
|
raise SupersetAPIError(f"API Error {status_code}: {exc.response.text}") from exc
|
||||||
# [/DEF:_handle_http_error:Function]
|
# [/DEF:AsyncAPIClient._handle_http_error:Function]
|
||||||
|
|
||||||
# [DEF:_handle_network_error:Function]
|
# [DEF:AsyncAPIClient._is_dashboard_endpoint:Function]
|
||||||
|
# @COMPLEXITY: 2
|
||||||
|
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
|
||||||
|
# @POST: Returns true only for dashboard-specific endpoints.
|
||||||
|
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
|
||||||
|
normalized_endpoint = str(endpoint or "").strip().lower()
|
||||||
|
if not normalized_endpoint:
|
||||||
|
return False
|
||||||
|
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
|
||||||
|
try:
|
||||||
|
normalized_endpoint = "/" + normalized_endpoint.split("/api/v1", 1)[1].lstrip("/")
|
||||||
|
except IndexError:
|
||||||
|
return False
|
||||||
|
if normalized_endpoint.startswith("/api/v1/"):
|
||||||
|
normalized_endpoint = normalized_endpoint[len("/api/v1"):]
|
||||||
|
return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard"
|
||||||
|
# [/DEF:backend.src.core.utils.async_network.AsyncAPIClient._is_dashboard_endpoint:Function]
|
||||||
|
|
||||||
|
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient._handle_network_error:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Translate generic httpx errors into NetworkError.
|
# @PURPOSE: Translate generic httpx errors into NetworkError.
|
||||||
# @POST: Raises NetworkError with URL context.
|
# @POST: Raises NetworkError with URL context.
|
||||||
@@ -251,16 +280,16 @@ class AsyncAPIClient:
|
|||||||
else:
|
else:
|
||||||
message = f"Unknown network error: {exc}"
|
message = f"Unknown network error: {exc}"
|
||||||
raise NetworkError(message, url=url) from exc
|
raise NetworkError(message, url=url) from exc
|
||||||
# [/DEF:_handle_network_error:Function]
|
# [/DEF:backend.src.core.utils.async_network.AsyncAPIClient._handle_network_error:Function]
|
||||||
|
|
||||||
# [DEF:aclose:Function]
|
# [DEF:backend.src.core.utils.async_network.AsyncAPIClient.aclose:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Close underlying httpx client.
|
# @PURPOSE: Close underlying httpx client.
|
||||||
# @POST: Client resources are released.
|
# @POST: Client resources are released.
|
||||||
# @SIDE_EFFECT: Closes network connections.
|
# @SIDE_EFFECT: Closes network connections.
|
||||||
async def aclose(self) -> None:
|
async def aclose(self) -> None:
|
||||||
await self._client.aclose()
|
await self._client.aclose()
|
||||||
# [/DEF:aclose:Function]
|
# [/DEF:backend.src.core.utils.async_network.AsyncAPIClient.aclose:Function]
|
||||||
# [/DEF:AsyncAPIClient:Class]
|
# [/DEF:backend.src.core.utils.async_network.AsyncAPIClient:Class]
|
||||||
|
|
||||||
# [/DEF:backend.src.core.utils.async_network:Module]
|
# [/DEF:backend.src.core.utils.async_network:Module]
|
||||||
|
|||||||
@@ -1,11 +1,10 @@
|
|||||||
# [DEF:network:Module]
|
# [DEF:NetworkModule:Module]
|
||||||
#
|
#
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @SEMANTICS: network, http, client, api, requests, session, authentication
|
# @SEMANTICS: network, http, client, api, requests, session, authentication
|
||||||
# @PURPOSE: Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
|
# @PURPOSE: Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
|
||||||
# @LAYER: Infra
|
# @LAYER: Infra
|
||||||
# @RELATION: DEPENDS_ON -> backend.src.core.logger
|
# @RELATION: [DEPENDS_ON] ->[LoggerModule]
|
||||||
# @RELATION: DEPENDS_ON -> requests
|
|
||||||
# @PUBLIC_API: APIClient
|
# @PUBLIC_API: APIClient
|
||||||
|
|
||||||
# [SECTION: IMPORTS]
|
# [SECTION: IMPORTS]
|
||||||
@@ -82,7 +81,7 @@ class DashboardNotFoundError(SupersetAPIError):
|
|||||||
# [DEF:NetworkError:Class]
|
# [DEF:NetworkError:Class]
|
||||||
# @PURPOSE: Exception raised when a network level error occurs.
|
# @PURPOSE: Exception raised when a network level error occurs.
|
||||||
class NetworkError(Exception):
|
class NetworkError(Exception):
|
||||||
# [DEF:network.APIClient.__init__:Function]
|
# [DEF:NetworkError.__init__:Function]
|
||||||
# @PURPOSE: Initializes the network error.
|
# @PURPOSE: Initializes the network error.
|
||||||
# @PRE: message is a string.
|
# @PRE: message is a string.
|
||||||
# @POST: NetworkError is initialized.
|
# @POST: NetworkError is initialized.
|
||||||
@@ -90,11 +89,11 @@ class NetworkError(Exception):
|
|||||||
with belief_scope("NetworkError.__init__"):
|
with belief_scope("NetworkError.__init__"):
|
||||||
self.context = context
|
self.context = context
|
||||||
super().__init__(f"[NETWORK_FAILURE] {message} | Context: {self.context}")
|
super().__init__(f"[NETWORK_FAILURE] {message} | Context: {self.context}")
|
||||||
# [/DEF:__init__:Function]
|
# [/DEF:NetworkError.__init__:Function]
|
||||||
# [/DEF:NetworkError:Class]
|
# [/DEF:NetworkError:Class]
|
||||||
|
|
||||||
|
|
||||||
# [DEF:network.SupersetAuthCache:Class]
|
# [DEF:SupersetAuthCache:Class]
|
||||||
# @PURPOSE: Process-local cache for Superset access/csrf tokens keyed by environment credentials.
|
# @PURPOSE: Process-local cache for Superset access/csrf tokens keyed by environment credentials.
|
||||||
# @PRE: base_url and username are stable strings.
|
# @PRE: base_url and username are stable strings.
|
||||||
# @POST: Cached entries expire automatically by TTL and can be reused across requests.
|
# @POST: Cached entries expire automatically by TTL and can be reused across requests.
|
||||||
@@ -152,8 +151,8 @@ class SupersetAuthCache:
|
|||||||
# [DEF:APIClient:Class]
|
# [DEF:APIClient:Class]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Synchronous Superset API client with process-local auth token caching.
|
# @PURPOSE: Synchronous Superset API client with process-local auth token caching.
|
||||||
# @RELATION: DEPENDS_ON -> network.SupersetAuthCache
|
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
|
||||||
# @RELATION: DEPENDS_ON -> logger
|
# @RELATION: [DEPENDS_ON] ->[LoggerModule]
|
||||||
class APIClient:
|
class APIClient:
|
||||||
DEFAULT_TIMEOUT = 30
|
DEFAULT_TIMEOUT = 30
|
||||||
|
|
||||||
@@ -256,7 +255,7 @@ class APIClient:
|
|||||||
return f"{self.api_base_url}{normalized_endpoint}"
|
return f"{self.api_base_url}{normalized_endpoint}"
|
||||||
# [/DEF:_build_api_url:Function]
|
# [/DEF:_build_api_url:Function]
|
||||||
|
|
||||||
# [DEF:authenticate:Function]
|
# [DEF:APIClient.authenticate:Function]
|
||||||
# @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.
|
# @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.
|
||||||
# @PRE: self.auth and self.base_url must be valid.
|
# @PRE: self.auth and self.base_url must be valid.
|
||||||
# @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.
|
# @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.
|
||||||
@@ -364,7 +363,14 @@ class APIClient:
|
|||||||
if status_code == 502 or status_code == 503 or status_code == 504:
|
if status_code == 502 or status_code == 503 or status_code == 504:
|
||||||
raise NetworkError(f"Environment unavailable (Status {status_code})", status_code=status_code) from e
|
raise NetworkError(f"Environment unavailable (Status {status_code})", status_code=status_code) from e
|
||||||
if status_code == 404:
|
if status_code == 404:
|
||||||
raise DashboardNotFoundError(endpoint) from e
|
if self._is_dashboard_endpoint(endpoint):
|
||||||
|
raise DashboardNotFoundError(endpoint) from e
|
||||||
|
raise SupersetAPIError(
|
||||||
|
f"API resource not found at endpoint '{endpoint}'",
|
||||||
|
status_code=status_code,
|
||||||
|
endpoint=endpoint,
|
||||||
|
subtype="not_found",
|
||||||
|
) from e
|
||||||
if status_code == 403:
|
if status_code == 403:
|
||||||
raise PermissionDeniedError() from e
|
raise PermissionDeniedError() from e
|
||||||
if status_code == 401:
|
if status_code == 401:
|
||||||
@@ -372,6 +378,24 @@ class APIClient:
|
|||||||
raise SupersetAPIError(f"API Error {status_code}: {e.response.text}") from e
|
raise SupersetAPIError(f"API Error {status_code}: {e.response.text}") from e
|
||||||
# [/DEF:_handle_http_error:Function]
|
# [/DEF:_handle_http_error:Function]
|
||||||
|
|
||||||
|
# [DEF:_is_dashboard_endpoint:Function]
|
||||||
|
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
|
||||||
|
# @PRE: endpoint may be relative or absolute.
|
||||||
|
# @POST: Returns true only for dashboard-specific endpoints.
|
||||||
|
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
|
||||||
|
normalized_endpoint = str(endpoint or "").strip().lower()
|
||||||
|
if not normalized_endpoint:
|
||||||
|
return False
|
||||||
|
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
|
||||||
|
try:
|
||||||
|
normalized_endpoint = "/" + normalized_endpoint.split("/api/v1", 1)[1].lstrip("/")
|
||||||
|
except IndexError:
|
||||||
|
return False
|
||||||
|
if normalized_endpoint.startswith("/api/v1/"):
|
||||||
|
normalized_endpoint = normalized_endpoint[len("/api/v1"):]
|
||||||
|
return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard"
|
||||||
|
# [/DEF:_is_dashboard_endpoint:Function]
|
||||||
|
|
||||||
# [DEF:_handle_network_error:Function]
|
# [DEF:_handle_network_error:Function]
|
||||||
# @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.
|
# @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.
|
||||||
# @PARAM: e (requests.exceptions.RequestException) - Ошибка.
|
# @PARAM: e (requests.exceptions.RequestException) - Ошибка.
|
||||||
@@ -505,4 +529,4 @@ class APIClient:
|
|||||||
|
|
||||||
# [/DEF:APIClient:Class]
|
# [/DEF:APIClient:Class]
|
||||||
|
|
||||||
# [/DEF:backend.core.utils.network:Module]
|
# [/DEF:NetworkModule:Module]
|
||||||
|
|||||||
@@ -3,9 +3,8 @@
|
|||||||
# @SEMANTICS: dataset_review, superset, compilation_preview, sql_lab_launch, execution_truth
|
# @SEMANTICS: dataset_review, superset, compilation_preview, sql_lab_launch, execution_truth
|
||||||
# @PURPOSE: Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context.
|
# @PURPOSE: Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context.
|
||||||
# @LAYER: Infra
|
# @LAYER: Infra
|
||||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
# @RELATION: [CALLS] ->[SupersetClient]
|
||||||
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
|
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
|
||||||
# @RELATION: [DEPENDS_ON] ->[DatasetRunContext]
|
|
||||||
# @PRE: effective template params and dataset execution reference are available.
|
# @PRE: effective template params and dataset execution reference are available.
|
||||||
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
|
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
|
||||||
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
|
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
|
||||||
@@ -54,7 +53,7 @@ class SqlLabLaunchPayload:
|
|||||||
# [DEF:SupersetCompilationAdapter:Class]
|
# [DEF:SupersetCompilationAdapter:Class]
|
||||||
# @COMPLEXITY: 4
|
# @COMPLEXITY: 4
|
||||||
# @PURPOSE: Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
|
# @PURPOSE: Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
|
||||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
# @RELATION: [CALLS] ->[SupersetClient]
|
||||||
# @PRE: environment is configured and Superset is reachable for the target session.
|
# @PRE: environment is configured and Superset is reachable for the target session.
|
||||||
# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
|
# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
|
||||||
# @SIDE_EFFECT: issues network requests to Superset API surfaces.
|
# @SIDE_EFFECT: issues network requests to Superset API surfaces.
|
||||||
@@ -222,57 +221,49 @@ class SupersetCompilationAdapter:
|
|||||||
|
|
||||||
# [DEF:SupersetCompilationAdapter._request_superset_preview:Function]
|
# [DEF:SupersetCompilationAdapter._request_superset_preview:Function]
|
||||||
# @COMPLEXITY: 4
|
# @COMPLEXITY: 4
|
||||||
# @PURPOSE: Probe supported Superset preview surfaces and return the first explicit compilation response.
|
# @PURPOSE: Request preview compilation through explicit client support backed by real Superset endpoints only.
|
||||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
# @RELATION: [CALLS] ->[SupersetClient.compile_dataset_preview]
|
||||||
# @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
|
# @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
|
||||||
# @POST: returns the first upstream response that exposes compiled SQL without fabricating local SQL.
|
# @POST: returns one normalized upstream compilation response without endpoint guessing.
|
||||||
# @SIDE_EFFECT: issues one or more Superset preview requests until a supported surface responds.
|
# @SIDE_EFFECT: issues one Superset chart-data request through the client.
|
||||||
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
|
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
|
||||||
def _request_superset_preview(self, payload: PreviewCompilationPayload) -> Dict[str, Any]:
|
def _request_superset_preview(self, payload: PreviewCompilationPayload) -> Dict[str, Any]:
|
||||||
request_payload = {
|
try:
|
||||||
"dataset_id": payload.dataset_id,
|
logger.reason(
|
||||||
"template_params": payload.template_params,
|
"Attempting deterministic Superset preview compilation via chart/data",
|
||||||
"effective_filters": payload.effective_filters,
|
extra={
|
||||||
"session_id": payload.session_id,
|
"dataset_id": payload.dataset_id,
|
||||||
}
|
"session_id": payload.session_id,
|
||||||
candidate_calls = self._build_preview_call_candidates(payload.dataset_id, request_payload)
|
"filter_count": len(payload.effective_filters),
|
||||||
errors: List[str] = []
|
"template_param_count": len(payload.template_params),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
response = self.client.compile_dataset_preview(
|
||||||
|
dataset_id=payload.dataset_id,
|
||||||
|
template_params=payload.template_params,
|
||||||
|
effective_filters=payload.effective_filters,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.explore(
|
||||||
|
"Superset preview compilation via chart/data failed",
|
||||||
|
extra={
|
||||||
|
"dataset_id": payload.dataset_id,
|
||||||
|
"session_id": payload.session_id,
|
||||||
|
"error": str(exc),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
raise RuntimeError(str(exc)) from exc
|
||||||
|
|
||||||
for candidate in candidate_calls:
|
normalized = self._normalize_preview_response(response)
|
||||||
call_kind = candidate["kind"]
|
if normalized is None:
|
||||||
target = candidate["target"]
|
raise RuntimeError("Superset chart/data compilation response could not be normalized")
|
||||||
try:
|
return normalized
|
||||||
logger.reason(
|
|
||||||
"Attempting Superset preview compilation candidate",
|
|
||||||
extra={"kind": call_kind, "target": target},
|
|
||||||
)
|
|
||||||
if call_kind == "client_method":
|
|
||||||
method = getattr(self.client, target)
|
|
||||||
response = method(request_payload)
|
|
||||||
else:
|
|
||||||
response = self.client.network.request(
|
|
||||||
method=candidate["http_method"],
|
|
||||||
endpoint=target,
|
|
||||||
data=candidate["data"],
|
|
||||||
headers={"Content-Type": "application/json"},
|
|
||||||
)
|
|
||||||
normalized = self._normalize_preview_response(response)
|
|
||||||
if normalized is not None:
|
|
||||||
return normalized
|
|
||||||
except Exception as exc:
|
|
||||||
errors.append(f"{call_kind}:{target}:{exc}")
|
|
||||||
logger.explore(
|
|
||||||
"Superset preview compilation candidate failed",
|
|
||||||
extra={"kind": call_kind, "target": target, "error": str(exc)},
|
|
||||||
)
|
|
||||||
|
|
||||||
raise RuntimeError("; ".join(errors) or "No Superset preview surface accepted the request")
|
|
||||||
# [/DEF:SupersetCompilationAdapter._request_superset_preview:Function]
|
# [/DEF:SupersetCompilationAdapter._request_superset_preview:Function]
|
||||||
|
|
||||||
# [DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
|
# [DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
|
||||||
# @COMPLEXITY: 4
|
# @COMPLEXITY: 4
|
||||||
# @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response.
|
# @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response.
|
||||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
# @RELATION: [CALLS] ->[SupersetClient.get_dataset]
|
||||||
# @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
|
# @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
|
||||||
# @POST: returns the first successful SQL Lab execution response from Superset.
|
# @POST: returns the first successful SQL Lab execution response from Superset.
|
||||||
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
|
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
|
||||||
@@ -317,49 +308,6 @@ class SupersetCompilationAdapter:
|
|||||||
raise RuntimeError("; ".join(errors) or "No Superset SQL Lab surface accepted the request")
|
raise RuntimeError("; ".join(errors) or "No Superset SQL Lab surface accepted the request")
|
||||||
# [/DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
|
# [/DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
|
||||||
|
|
||||||
# [DEF:SupersetCompilationAdapter._build_preview_call_candidates:Function]
|
|
||||||
# @COMPLEXITY: 2
|
|
||||||
# @PURPOSE: Assemble preview candidate call shapes in priority order.
|
|
||||||
def _build_preview_call_candidates(
|
|
||||||
self,
|
|
||||||
dataset_id: int,
|
|
||||||
request_payload: Dict[str, Any],
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
candidates: List[Dict[str, Any]] = []
|
|
||||||
for method_name in (
|
|
||||||
"compile_sql_preview",
|
|
||||||
"compile_preview",
|
|
||||||
"get_compiled_sql_preview",
|
|
||||||
):
|
|
||||||
if hasattr(self.client, method_name):
|
|
||||||
candidates.append({"kind": "client_method", "target": method_name})
|
|
||||||
|
|
||||||
encoded_payload = self._dump_json(request_payload)
|
|
||||||
candidates.extend(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"kind": "network",
|
|
||||||
"target": f"/dataset/{dataset_id}/preview",
|
|
||||||
"http_method": "POST",
|
|
||||||
"data": encoded_payload,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"kind": "network",
|
|
||||||
"target": f"/dataset/{dataset_id}/sql",
|
|
||||||
"http_method": "POST",
|
|
||||||
"data": encoded_payload,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"kind": "network",
|
|
||||||
"target": "/sqllab/format_sql/",
|
|
||||||
"http_method": "POST",
|
|
||||||
"data": encoded_payload,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
)
|
|
||||||
return candidates
|
|
||||||
# [/DEF:SupersetCompilationAdapter._build_preview_call_candidates:Function]
|
|
||||||
|
|
||||||
# [DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
|
# [DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
|
||||||
# @COMPLEXITY: 3
|
# @COMPLEXITY: 3
|
||||||
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
|
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
|
||||||
|
|||||||
@@ -94,6 +94,8 @@ class SupersetContextExtractor:
|
|||||||
|
|
||||||
dataset_id = self._extract_numeric_identifier(path_parts, "dataset")
|
dataset_id = self._extract_numeric_identifier(path_parts, "dataset")
|
||||||
dashboard_id = self._extract_numeric_identifier(path_parts, "dashboard")
|
dashboard_id = self._extract_numeric_identifier(path_parts, "dashboard")
|
||||||
|
dashboard_ref = self._extract_dashboard_reference(path_parts)
|
||||||
|
dashboard_permalink_key = self._extract_dashboard_permalink_key(path_parts)
|
||||||
chart_id = self._extract_numeric_identifier(path_parts, "chart")
|
chart_id = self._extract_numeric_identifier(path_parts, "chart")
|
||||||
|
|
||||||
resource_type = "unknown"
|
resource_type = "unknown"
|
||||||
@@ -108,13 +110,82 @@ class SupersetContextExtractor:
|
|||||||
"Resolved direct dataset link",
|
"Resolved direct dataset link",
|
||||||
extra={"dataset_id": dataset_id},
|
extra={"dataset_id": dataset_id},
|
||||||
)
|
)
|
||||||
elif dashboard_id is not None:
|
elif dashboard_permalink_key is not None:
|
||||||
resource_type = "dashboard"
|
resource_type = "dashboard"
|
||||||
|
partial_recovery = True
|
||||||
|
dataset_ref = f"dashboard_permalink:{dashboard_permalink_key}"
|
||||||
|
unresolved_references.append("dashboard_permalink_dataset_binding_unresolved")
|
||||||
|
logger.reason(
|
||||||
|
"Resolving dashboard permalink state from Superset",
|
||||||
|
extra={"permalink_key": dashboard_permalink_key},
|
||||||
|
)
|
||||||
|
permalink_payload = self.client.get_dashboard_permalink_state(dashboard_permalink_key)
|
||||||
|
permalink_state = (
|
||||||
|
permalink_payload.get("state", permalink_payload)
|
||||||
|
if isinstance(permalink_payload, dict)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
if isinstance(permalink_state, dict):
|
||||||
|
for key, value in permalink_state.items():
|
||||||
|
query_state.setdefault(key, value)
|
||||||
|
resolved_dashboard_id = self._extract_dashboard_id_from_state(permalink_state)
|
||||||
|
resolved_chart_id = self._extract_chart_id_from_state(permalink_state)
|
||||||
|
if resolved_dashboard_id is not None:
|
||||||
|
dashboard_id = resolved_dashboard_id
|
||||||
|
unresolved_references = [
|
||||||
|
item
|
||||||
|
for item in unresolved_references
|
||||||
|
if item != "dashboard_permalink_dataset_binding_unresolved"
|
||||||
|
]
|
||||||
|
dataset_id, unresolved_references = self._recover_dataset_binding_from_dashboard(
|
||||||
|
dashboard_id=dashboard_id,
|
||||||
|
dataset_ref=dataset_ref,
|
||||||
|
unresolved_references=unresolved_references,
|
||||||
|
)
|
||||||
|
if dataset_id is not None:
|
||||||
|
dataset_ref = f"dataset:{dataset_id}"
|
||||||
|
elif resolved_chart_id is not None:
|
||||||
|
chart_id = resolved_chart_id
|
||||||
|
unresolved_references = [
|
||||||
|
item
|
||||||
|
for item in unresolved_references
|
||||||
|
if item != "dashboard_permalink_dataset_binding_unresolved"
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
chart_payload = self.client.get_chart(chart_id)
|
||||||
|
chart_data = chart_payload.get("result", chart_payload) if isinstance(chart_payload, dict) else {}
|
||||||
|
datasource_id = chart_data.get("datasource_id")
|
||||||
|
if datasource_id is not None:
|
||||||
|
dataset_id = int(datasource_id)
|
||||||
|
dataset_ref = f"dataset:{dataset_id}"
|
||||||
|
logger.reason(
|
||||||
|
"Recovered dataset reference from permalink chart context",
|
||||||
|
extra={"chart_id": chart_id, "dataset_id": dataset_id},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
unresolved_references.append("chart_dataset_binding_unresolved")
|
||||||
|
except Exception as exc:
|
||||||
|
unresolved_references.append("chart_dataset_binding_unresolved")
|
||||||
|
logger.explore(
|
||||||
|
"Chart lookup failed during permalink recovery",
|
||||||
|
extra={"chart_id": chart_id, "error": str(exc)},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.explore(
|
||||||
|
"Dashboard permalink state was not a structured object",
|
||||||
|
extra={"permalink_key": dashboard_permalink_key},
|
||||||
|
)
|
||||||
|
elif dashboard_id is not None or dashboard_ref is not None:
|
||||||
|
resource_type = "dashboard"
|
||||||
|
resolved_dashboard_ref = dashboard_id if dashboard_id is not None else dashboard_ref
|
||||||
logger.reason(
|
logger.reason(
|
||||||
"Resolving dashboard-bound dataset from Superset",
|
"Resolving dashboard-bound dataset from Superset",
|
||||||
extra={"dashboard_id": dashboard_id},
|
extra={"dashboard_ref": resolved_dashboard_ref},
|
||||||
)
|
)
|
||||||
dashboard_detail = self.client.get_dashboard_detail(dashboard_id)
|
dashboard_detail = self.client.get_dashboard_detail(resolved_dashboard_ref)
|
||||||
|
resolved_dashboard_id = dashboard_detail.get("id")
|
||||||
|
if resolved_dashboard_id is not None:
|
||||||
|
dashboard_id = int(resolved_dashboard_id)
|
||||||
datasets = dashboard_detail.get("datasets") or []
|
datasets = dashboard_detail.get("datasets") or []
|
||||||
if datasets:
|
if datasets:
|
||||||
first_dataset = datasets[0]
|
first_dataset = datasets[0]
|
||||||
@@ -460,6 +531,127 @@ class SupersetContextExtractor:
|
|||||||
return int(candidate)
|
return int(candidate)
|
||||||
# [/DEF:SupersetContextExtractor._extract_numeric_identifier:Function]
|
# [/DEF:SupersetContextExtractor._extract_numeric_identifier:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetContextExtractor._extract_dashboard_reference:Function]
|
||||||
|
# @COMPLEXITY: 2
|
||||||
|
# @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path.
|
||||||
|
def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]:
|
||||||
|
if "dashboard" not in path_parts:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
resource_index = path_parts.index("dashboard")
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if resource_index + 1 >= len(path_parts):
|
||||||
|
return None
|
||||||
|
|
||||||
|
candidate = str(path_parts[resource_index + 1]).strip()
|
||||||
|
if not candidate or candidate == "p":
|
||||||
|
return None
|
||||||
|
return candidate
|
||||||
|
# [/DEF:SupersetContextExtractor._extract_dashboard_reference:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetContextExtractor._extract_dashboard_permalink_key:Function]
|
||||||
|
# @COMPLEXITY: 2
|
||||||
|
# @PURPOSE: Extract a dashboard permalink key from a Superset URL path.
|
||||||
|
def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]:
|
||||||
|
if "dashboard" not in path_parts:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
resource_index = path_parts.index("dashboard")
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if resource_index + 2 >= len(path_parts):
|
||||||
|
return None
|
||||||
|
|
||||||
|
permalink_marker = str(path_parts[resource_index + 1]).strip()
|
||||||
|
permalink_key = str(path_parts[resource_index + 2]).strip()
|
||||||
|
if permalink_marker != "p" or not permalink_key:
|
||||||
|
return None
|
||||||
|
return permalink_key
|
||||||
|
# [/DEF:SupersetContextExtractor._extract_dashboard_permalink_key:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetContextExtractor._extract_dashboard_id_from_state:Function]
|
||||||
|
# @COMPLEXITY: 2
|
||||||
|
# @PURPOSE: Extract a dashboard identifier from returned permalink state when present.
|
||||||
|
def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
|
||||||
|
return self._search_nested_numeric_key(
|
||||||
|
payload=state,
|
||||||
|
candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"},
|
||||||
|
)
|
||||||
|
# [/DEF:SupersetContextExtractor._extract_dashboard_id_from_state:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetContextExtractor._extract_chart_id_from_state:Function]
|
||||||
|
# @COMPLEXITY: 2
|
||||||
|
# @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent.
|
||||||
|
def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
|
||||||
|
return self._search_nested_numeric_key(
|
||||||
|
payload=state,
|
||||||
|
candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"},
|
||||||
|
)
|
||||||
|
# [/DEF:SupersetContextExtractor._extract_chart_id_from_state:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetContextExtractor._search_nested_numeric_key:Function]
|
||||||
|
# @COMPLEXITY: 3
|
||||||
|
# @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
|
||||||
|
def _search_nested_numeric_key(self, payload: Any, candidate_keys: Set[str]) -> Optional[int]:
|
||||||
|
if isinstance(payload, dict):
|
||||||
|
for key, value in payload.items():
|
||||||
|
if key in candidate_keys:
|
||||||
|
try:
|
||||||
|
if value is not None:
|
||||||
|
return int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
found = self._search_nested_numeric_key(value, candidate_keys)
|
||||||
|
if found is not None:
|
||||||
|
return found
|
||||||
|
elif isinstance(payload, list):
|
||||||
|
for item in payload:
|
||||||
|
found = self._search_nested_numeric_key(item, candidate_keys)
|
||||||
|
if found is not None:
|
||||||
|
return found
|
||||||
|
return None
|
||||||
|
# [/DEF:SupersetContextExtractor._search_nested_numeric_key:Function]
|
||||||
|
|
||||||
|
# [DEF:SupersetContextExtractor._recover_dataset_binding_from_dashboard:Function]
|
||||||
|
# @COMPLEXITY: 3
|
||||||
|
# @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers.
|
||||||
|
def _recover_dataset_binding_from_dashboard(
|
||||||
|
self,
|
||||||
|
dashboard_id: int,
|
||||||
|
dataset_ref: Optional[str],
|
||||||
|
unresolved_references: List[str],
|
||||||
|
) -> tuple[Optional[int], List[str]]:
|
||||||
|
dashboard_detail = self.client.get_dashboard_detail(dashboard_id)
|
||||||
|
datasets = dashboard_detail.get("datasets") or []
|
||||||
|
if datasets:
|
||||||
|
first_dataset = datasets[0]
|
||||||
|
resolved_dataset_id = first_dataset.get("id")
|
||||||
|
if resolved_dataset_id is not None:
|
||||||
|
resolved_dataset = int(resolved_dataset_id)
|
||||||
|
logger.reason(
|
||||||
|
"Recovered dataset reference from dashboard permalink context",
|
||||||
|
extra={
|
||||||
|
"dashboard_id": dashboard_id,
|
||||||
|
"dataset_id": resolved_dataset,
|
||||||
|
"dataset_count": len(datasets),
|
||||||
|
"dataset_ref": dataset_ref,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if len(datasets) > 1 and "multiple_dashboard_datasets" not in unresolved_references:
|
||||||
|
unresolved_references.append("multiple_dashboard_datasets")
|
||||||
|
return resolved_dataset, unresolved_references
|
||||||
|
if "dashboard_dataset_id_missing" not in unresolved_references:
|
||||||
|
unresolved_references.append("dashboard_dataset_id_missing")
|
||||||
|
return None, unresolved_references
|
||||||
|
|
||||||
|
if "dashboard_dataset_binding_missing" not in unresolved_references:
|
||||||
|
unresolved_references.append("dashboard_dataset_binding_missing")
|
||||||
|
return None, unresolved_references
|
||||||
|
# [/DEF:SupersetContextExtractor._recover_dataset_binding_from_dashboard:Function]
|
||||||
|
|
||||||
# [DEF:SupersetContextExtractor._decode_query_state:Function]
|
# [DEF:SupersetContextExtractor._decode_query_state:Function]
|
||||||
# @COMPLEXITY: 2
|
# @COMPLEXITY: 2
|
||||||
# @PURPOSE: Decode query-string structures used by Superset URL state transport.
|
# @PURPOSE: Decode query-string structures used by Superset URL state transport.
|
||||||
@@ -470,7 +662,7 @@ class SupersetContextExtractor:
|
|||||||
continue
|
continue
|
||||||
raw_value = values[-1]
|
raw_value = values[-1]
|
||||||
decoded_value = unquote(raw_value)
|
decoded_value = unquote(raw_value)
|
||||||
if key in {"native_filters", "native_filters_key", "form_data", "q"}:
|
if key in {"native_filters", "form_data", "q"}:
|
||||||
try:
|
try:
|
||||||
query_state[key] = json.loads(decoded_value)
|
query_state[key] = json.loads(decoded_value)
|
||||||
continue
|
continue
|
||||||
@@ -514,6 +706,36 @@ class SupersetContextExtractor:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
dashboard_data_mask = query_state.get("dataMask")
|
||||||
|
if isinstance(dashboard_data_mask, dict):
|
||||||
|
for filter_key, item in dashboard_data_mask.items():
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
filter_state = item.get("filterState")
|
||||||
|
extra_form_data = item.get("extraFormData")
|
||||||
|
display_name = None
|
||||||
|
raw_value = None
|
||||||
|
if isinstance(filter_state, dict):
|
||||||
|
display_name = filter_state.get("label")
|
||||||
|
raw_value = filter_state.get("value")
|
||||||
|
if raw_value is None and isinstance(extra_form_data, dict):
|
||||||
|
extra_filters = extra_form_data.get("filters")
|
||||||
|
if isinstance(extra_filters, list) and extra_filters:
|
||||||
|
first_filter = extra_filters[0]
|
||||||
|
if isinstance(first_filter, dict):
|
||||||
|
raw_value = first_filter.get("val")
|
||||||
|
imported_filters.append(
|
||||||
|
{
|
||||||
|
"filter_name": str(item.get("id") or filter_key),
|
||||||
|
"raw_value": raw_value,
|
||||||
|
"display_name": display_name,
|
||||||
|
"source": "superset_permalink",
|
||||||
|
"recovery_status": "recovered" if raw_value is not None else "partial",
|
||||||
|
"requires_confirmation": raw_value is None,
|
||||||
|
"notes": "Recovered from Superset dashboard permalink state",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
form_data_payload = query_state.get("form_data")
|
form_data_payload = query_state.get("form_data")
|
||||||
if isinstance(form_data_payload, dict):
|
if isinstance(form_data_payload, dict):
|
||||||
extra_filters = form_data_payload.get("extra_filters") or []
|
extra_filters = form_data_payload.get("extra_filters") or []
|
||||||
|
|||||||
@@ -45,16 +45,25 @@ from src.models.dataset_review import (
|
|||||||
DatasetProfile,
|
DatasetProfile,
|
||||||
DatasetReviewSession,
|
DatasetReviewSession,
|
||||||
DatasetRunContext,
|
DatasetRunContext,
|
||||||
|
ExecutionMapping,
|
||||||
|
FilterConfidenceState,
|
||||||
|
FilterRecoveryStatus,
|
||||||
|
FilterSource,
|
||||||
FindingArea,
|
FindingArea,
|
||||||
FindingSeverity,
|
FindingSeverity,
|
||||||
|
ImportedFilter,
|
||||||
LaunchStatus,
|
LaunchStatus,
|
||||||
|
MappingMethod,
|
||||||
|
MappingStatus,
|
||||||
PreviewStatus,
|
PreviewStatus,
|
||||||
RecommendedAction,
|
RecommendedAction,
|
||||||
ReadinessState,
|
ReadinessState,
|
||||||
ResolutionState,
|
ResolutionState,
|
||||||
SessionPhase,
|
SessionPhase,
|
||||||
SessionStatus,
|
SessionStatus,
|
||||||
|
TemplateVariable,
|
||||||
ValidationFinding,
|
ValidationFinding,
|
||||||
|
VariableKind,
|
||||||
)
|
)
|
||||||
from src.services.dataset_review.repositories.session_repository import (
|
from src.services.dataset_review.repositories.session_repository import (
|
||||||
DatasetReviewSessionRepository,
|
DatasetReviewSessionRepository,
|
||||||
@@ -248,6 +257,17 @@ class DatasetReviewOrchestrator:
|
|||||||
)
|
)
|
||||||
persisted_session = self.repository.create_session(session)
|
persisted_session = self.repository.create_session(session)
|
||||||
|
|
||||||
|
recovered_filters: List[ImportedFilter] = []
|
||||||
|
template_variables: List[TemplateVariable] = []
|
||||||
|
execution_mappings: List[ExecutionMapping] = []
|
||||||
|
if normalized_source_kind == "superset_link" and parsed_context is not None:
|
||||||
|
recovered_filters, template_variables, execution_mappings, findings = self._build_recovery_bootstrap(
|
||||||
|
environment=environment,
|
||||||
|
session=persisted_session,
|
||||||
|
parsed_context=parsed_context,
|
||||||
|
findings=findings,
|
||||||
|
)
|
||||||
|
|
||||||
profile = self._build_initial_profile(
|
profile = self._build_initial_profile(
|
||||||
session_id=persisted_session.session_id,
|
session_id=persisted_session.session_id,
|
||||||
parsed_context=parsed_context,
|
parsed_context=parsed_context,
|
||||||
@@ -276,6 +296,14 @@ class DatasetReviewOrchestrator:
|
|||||||
profile,
|
profile,
|
||||||
findings,
|
findings,
|
||||||
)
|
)
|
||||||
|
if recovered_filters or template_variables or execution_mappings:
|
||||||
|
persisted_session = self.repository.save_recovery_state(
|
||||||
|
persisted_session.session_id,
|
||||||
|
command.user.id,
|
||||||
|
recovered_filters,
|
||||||
|
template_variables,
|
||||||
|
execution_mappings,
|
||||||
|
)
|
||||||
|
|
||||||
active_task_id = self._enqueue_recovery_task(
|
active_task_id = self._enqueue_recovery_task(
|
||||||
command=command,
|
command=command,
|
||||||
@@ -644,6 +672,115 @@ class DatasetReviewOrchestrator:
|
|||||||
return findings
|
return findings
|
||||||
# [/DEF:DatasetReviewOrchestrator._build_partial_recovery_findings:Function]
|
# [/DEF:DatasetReviewOrchestrator._build_partial_recovery_findings:Function]
|
||||||
|
|
||||||
|
# [DEF:DatasetReviewOrchestrator._build_recovery_bootstrap:Function]
|
||||||
|
# @COMPLEXITY: 4
|
||||||
|
# @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation.
|
||||||
|
def _build_recovery_bootstrap(
|
||||||
|
self,
|
||||||
|
environment,
|
||||||
|
session: DatasetReviewSession,
|
||||||
|
parsed_context: SupersetParsedContext,
|
||||||
|
findings: List[ValidationFinding],
|
||||||
|
) -> tuple[List[ImportedFilter], List[TemplateVariable], List[ExecutionMapping], List[ValidationFinding]]:
|
||||||
|
extractor = SupersetContextExtractor(environment)
|
||||||
|
imported_filters_payload = extractor.recover_imported_filters(parsed_context)
|
||||||
|
if imported_filters_payload is None:
|
||||||
|
imported_filters_payload = []
|
||||||
|
imported_filters = [
|
||||||
|
ImportedFilter(
|
||||||
|
session_id=session.session_id,
|
||||||
|
filter_name=str(item.get("filter_name") or f"imported_filter_{index}"),
|
||||||
|
display_name=item.get("display_name"),
|
||||||
|
raw_value=item.get("raw_value"),
|
||||||
|
normalized_value=item.get("normalized_value"),
|
||||||
|
source=FilterSource(str(item.get("source") or FilterSource.SUPERSET_URL.value)),
|
||||||
|
confidence_state=FilterConfidenceState(
|
||||||
|
str(item.get("confidence_state") or FilterConfidenceState.UNRESOLVED.value)
|
||||||
|
),
|
||||||
|
requires_confirmation=bool(item.get("requires_confirmation", False)),
|
||||||
|
recovery_status=FilterRecoveryStatus(
|
||||||
|
str(item.get("recovery_status") or FilterRecoveryStatus.PARTIAL.value)
|
||||||
|
),
|
||||||
|
notes=item.get("notes"),
|
||||||
|
)
|
||||||
|
for index, item in enumerate(imported_filters_payload)
|
||||||
|
]
|
||||||
|
|
||||||
|
template_variables: List[TemplateVariable] = []
|
||||||
|
execution_mappings: List[ExecutionMapping] = []
|
||||||
|
|
||||||
|
if session.dataset_id is not None:
|
||||||
|
try:
|
||||||
|
dataset_payload = extractor.client.get_dataset_detail(session.dataset_id)
|
||||||
|
discovered_variables = extractor.discover_template_variables(dataset_payload)
|
||||||
|
template_variables = [
|
||||||
|
TemplateVariable(
|
||||||
|
session_id=session.session_id,
|
||||||
|
variable_name=str(item.get("variable_name") or f"variable_{index}"),
|
||||||
|
expression_source=str(item.get("expression_source") or ""),
|
||||||
|
variable_kind=VariableKind(str(item.get("variable_kind") or VariableKind.UNKNOWN.value)),
|
||||||
|
is_required=bool(item.get("is_required", True)),
|
||||||
|
default_value=item.get("default_value"),
|
||||||
|
mapping_status=MappingStatus(str(item.get("mapping_status") or MappingStatus.UNMAPPED.value)),
|
||||||
|
)
|
||||||
|
for index, item in enumerate(discovered_variables)
|
||||||
|
]
|
||||||
|
except Exception as exc:
|
||||||
|
if "dataset_template_variable_discovery_failed" not in parsed_context.unresolved_references:
|
||||||
|
parsed_context.unresolved_references.append("dataset_template_variable_discovery_failed")
|
||||||
|
if not any(
|
||||||
|
finding.caused_by_ref == "dataset_template_variable_discovery_failed"
|
||||||
|
for finding in findings
|
||||||
|
):
|
||||||
|
findings.append(
|
||||||
|
ValidationFinding(
|
||||||
|
area=FindingArea.TEMPLATE_MAPPING,
|
||||||
|
severity=FindingSeverity.WARNING,
|
||||||
|
code="TEMPLATE_VARIABLE_DISCOVERY_FAILED",
|
||||||
|
title="Template variables could not be discovered",
|
||||||
|
message="Session remains usable, but dataset template variables still need review.",
|
||||||
|
resolution_state=ResolutionState.OPEN,
|
||||||
|
caused_by_ref="dataset_template_variable_discovery_failed",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.explore(
|
||||||
|
"Template variable discovery failed during session bootstrap",
|
||||||
|
extra={"session_id": session.session_id, "dataset_id": session.dataset_id, "error": str(exc)},
|
||||||
|
)
|
||||||
|
|
||||||
|
filter_lookup = {
|
||||||
|
str(imported_filter.filter_name or "").strip().lower(): imported_filter
|
||||||
|
for imported_filter in imported_filters
|
||||||
|
if str(imported_filter.filter_name or "").strip()
|
||||||
|
}
|
||||||
|
for template_variable in template_variables:
|
||||||
|
matched_filter = filter_lookup.get(str(template_variable.variable_name or "").strip().lower())
|
||||||
|
if matched_filter is None:
|
||||||
|
continue
|
||||||
|
requires_explicit_approval = bool(
|
||||||
|
matched_filter.requires_confirmation
|
||||||
|
or matched_filter.recovery_status != FilterRecoveryStatus.RECOVERED
|
||||||
|
)
|
||||||
|
execution_mappings.append(
|
||||||
|
ExecutionMapping(
|
||||||
|
session_id=session.session_id,
|
||||||
|
filter_id=matched_filter.filter_id,
|
||||||
|
variable_id=template_variable.variable_id,
|
||||||
|
mapping_method=MappingMethod.DIRECT_MATCH,
|
||||||
|
raw_input_value=matched_filter.raw_value,
|
||||||
|
effective_value=matched_filter.normalized_value if matched_filter.normalized_value is not None else matched_filter.raw_value,
|
||||||
|
transformation_note="Bootstrapped from Superset recovery context",
|
||||||
|
warning_level=None if not requires_explicit_approval else None,
|
||||||
|
requires_explicit_approval=requires_explicit_approval,
|
||||||
|
approval_state=ApprovalState.PENDING if requires_explicit_approval else ApprovalState.NOT_REQUIRED,
|
||||||
|
approved_by_user_id=None,
|
||||||
|
approved_at=None,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return imported_filters, template_variables, execution_mappings, findings
|
||||||
|
# [/DEF:DatasetReviewOrchestrator._build_recovery_bootstrap:Function]
|
||||||
|
|
||||||
# [DEF:DatasetReviewOrchestrator._build_execution_snapshot:Function]
|
# [DEF:DatasetReviewOrchestrator._build_execution_snapshot:Function]
|
||||||
# @COMPLEXITY: 4
|
# @COMPLEXITY: 4
|
||||||
# @PURPOSE: Build effective filters, template params, approvals, and fingerprint for preview and launch gating.
|
# @PURPOSE: Build effective filters, template params, approvals, and fingerprint for preview and launch gating.
|
||||||
|
|||||||
@@ -23,9 +23,12 @@ from src.models.dataset_review import (
|
|||||||
ValidationFinding,
|
ValidationFinding,
|
||||||
CompiledPreview,
|
CompiledPreview,
|
||||||
DatasetRunContext,
|
DatasetRunContext,
|
||||||
|
ExecutionMapping,
|
||||||
|
ImportedFilter,
|
||||||
SemanticFieldEntry,
|
SemanticFieldEntry,
|
||||||
SessionCollaborator,
|
SessionCollaborator,
|
||||||
SessionEvent,
|
SessionEvent,
|
||||||
|
TemplateVariable,
|
||||||
)
|
)
|
||||||
from src.core.logger import belief_scope, logger
|
from src.core.logger import belief_scope, logger
|
||||||
from src.services.dataset_review.event_logger import SessionEventLogger
|
from src.services.dataset_review.event_logger import SessionEventLogger
|
||||||
@@ -202,6 +205,71 @@ class DatasetReviewSessionRepository:
|
|||||||
return self.load_session_detail(session_id, user_id)
|
return self.load_session_detail(session_id, user_id)
|
||||||
# [/DEF:save_prof_find:Function]
|
# [/DEF:save_prof_find:Function]
|
||||||
|
|
||||||
|
# [DEF:save_recovery_state:Function]
|
||||||
|
# @COMPLEXITY: 4
|
||||||
|
# @PURPOSE: Persist imported filters, template variables, and initial execution mappings for one owned session.
|
||||||
|
# @RELATION: [DEPENDS_ON] -> [ImportedFilter]
|
||||||
|
# @RELATION: [DEPENDS_ON] -> [TemplateVariable]
|
||||||
|
# @RELATION: [DEPENDS_ON] -> [ExecutionMapping]
|
||||||
|
def save_recovery_state(
|
||||||
|
self,
|
||||||
|
session_id: str,
|
||||||
|
user_id: str,
|
||||||
|
imported_filters: List[ImportedFilter],
|
||||||
|
template_variables: List[TemplateVariable],
|
||||||
|
execution_mappings: List[ExecutionMapping],
|
||||||
|
) -> DatasetReviewSession:
|
||||||
|
with belief_scope("DatasetReviewSessionRepository.save_recovery_state"):
|
||||||
|
session = self._get_owned_session(session_id, user_id)
|
||||||
|
logger.reason(
|
||||||
|
"Persisting dataset review recovery bootstrap state",
|
||||||
|
extra={
|
||||||
|
"session_id": session_id,
|
||||||
|
"user_id": user_id,
|
||||||
|
"imported_filters_count": len(imported_filters),
|
||||||
|
"template_variables_count": len(template_variables),
|
||||||
|
"execution_mappings_count": len(execution_mappings),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.db.query(ExecutionMapping).filter(
|
||||||
|
ExecutionMapping.session_id == session_id
|
||||||
|
).delete()
|
||||||
|
self.db.query(TemplateVariable).filter(
|
||||||
|
TemplateVariable.session_id == session_id
|
||||||
|
).delete()
|
||||||
|
self.db.query(ImportedFilter).filter(
|
||||||
|
ImportedFilter.session_id == session_id
|
||||||
|
).delete()
|
||||||
|
|
||||||
|
for imported_filter in imported_filters:
|
||||||
|
imported_filter.session_id = session_id
|
||||||
|
self.db.add(imported_filter)
|
||||||
|
|
||||||
|
for template_variable in template_variables:
|
||||||
|
template_variable.session_id = session_id
|
||||||
|
self.db.add(template_variable)
|
||||||
|
|
||||||
|
self.db.flush()
|
||||||
|
|
||||||
|
for execution_mapping in execution_mappings:
|
||||||
|
execution_mapping.session_id = session_id
|
||||||
|
self.db.add(execution_mapping)
|
||||||
|
|
||||||
|
self.db.commit()
|
||||||
|
logger.reflect(
|
||||||
|
"Dataset review recovery bootstrap state committed",
|
||||||
|
extra={
|
||||||
|
"session_id": session.session_id,
|
||||||
|
"user_id": user_id,
|
||||||
|
"imported_filters_count": len(imported_filters),
|
||||||
|
"template_variables_count": len(template_variables),
|
||||||
|
"execution_mappings_count": len(execution_mappings),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return self.load_session_detail(session_id, user_id)
|
||||||
|
# [/DEF:save_recovery_state:Function]
|
||||||
|
|
||||||
# [DEF:save_prev:Function]
|
# [DEF:save_prev:Function]
|
||||||
# @COMPLEXITY: 4
|
# @COMPLEXITY: 4
|
||||||
# @PURPOSE: Persist a preview snapshot and mark prior session previews stale.
|
# @PURPOSE: Persist a preview snapshot and mark prior session previews stale.
|
||||||
|
|||||||
@@ -3,9 +3,9 @@
|
|||||||
<!-- @SEMANTICS: dataset-review, workspace-entry, source-intake, session-bootstrap -->
|
<!-- @SEMANTICS: dataset-review, workspace-entry, source-intake, session-bootstrap -->
|
||||||
<!-- @PURPOSE: Entry route for Dataset Review Workspace that allows starting a new resumable review session before navigating to a specific session id route. -->
|
<!-- @PURPOSE: Entry route for Dataset Review Workspace that allows starting a new resumable review session before navigating to a specific session id route. -->
|
||||||
<!-- @LAYER: UI -->
|
<!-- @LAYER: UI -->
|
||||||
<!-- @RELATION: [CALLS] -> [api.postApi()](frontend/src/routes/datasets/review/+page.svelte:66) -->
|
<!-- @RELATION: [CALLS] ->[postApi] -->
|
||||||
<!-- @RELATION: [BINDS_TO] -> [SourceIntakePanel](frontend/src/lib/components/dataset-review/SourceIntakePanel.svelte) -->
|
<!-- @RELATION: [BINDS_TO] ->[SourceIntakePanel] -->
|
||||||
<!-- @RELATION: [BINDS_TO] -> [environmentContextStore](frontend/src/routes/datasets/review/+page.svelte:24) -->
|
<!-- @RELATION: [BINDS_TO] ->[environmentContext] -->
|
||||||
<!-- @UX_STATE: Empty -> Show source intake for Superset link or dataset reference. -->
|
<!-- @UX_STATE: Empty -> Show source intake for Superset link or dataset reference. -->
|
||||||
<!-- @UX_STATE: Submitting -> Disable controls and show startup feedback. -->
|
<!-- @UX_STATE: Submitting -> Disable controls and show startup feedback. -->
|
||||||
<!-- @UX_STATE: Error -> Inline error shown while keeping intake values editable. -->
|
<!-- @UX_STATE: Error -> Inline error shown while keeping intake values editable. -->
|
||||||
|
|||||||
@@ -188,7 +188,7 @@ This document defines the semantic contracts for the core components of the Data
|
|||||||
# @LAYER: Infra
|
# @LAYER: Infra
|
||||||
# @RELATION: [DEPENDS_ON] ->[ImportedFilter]
|
# @RELATION: [DEPENDS_ON] ->[ImportedFilter]
|
||||||
# @RELATION: [DEPENDS_ON] ->[TemplateVariable]
|
# @RELATION: [DEPENDS_ON] ->[TemplateVariable]
|
||||||
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
|
# @RELATION: [DEPENDS_ON] ->[SupersetClient]
|
||||||
# @DATA_CONTRACT: Input[SupersetLink | DatasetReference | EnvironmentContext] -> Output[RecoveredSupersetContext | ImportedFilterSet | TemplateVariableSet | RecoverySummary]
|
# @DATA_CONTRACT: Input[SupersetLink | DatasetReference | EnvironmentContext] -> Output[RecoveredSupersetContext | ImportedFilterSet | TemplateVariableSet | RecoverySummary]
|
||||||
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
|
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
|
||||||
# @POST: returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
|
# @POST: returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
|
||||||
@@ -223,7 +223,7 @@ This document defines the semantic contracts for the core components of the Data
|
|||||||
# @LAYER: Infra
|
# @LAYER: Infra
|
||||||
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
|
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
|
||||||
# @RELATION: [DEPENDS_ON] ->[DatasetRunContext]
|
# @RELATION: [DEPENDS_ON] ->[DatasetRunContext]
|
||||||
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
|
# @RELATION: [DEPENDS_ON] ->[SupersetClient]
|
||||||
# @DATA_CONTRACT: Input[ApprovedExecutionContext | PreviewFingerprint | LaunchRequest] -> Output[CompiledPreview | PreviewFailureArtifact | DatasetRunContext | LaunchFailureAudit]
|
# @DATA_CONTRACT: Input[ApprovedExecutionContext | PreviewFingerprint | LaunchRequest] -> Output[CompiledPreview | PreviewFailureArtifact | DatasetRunContext | LaunchFailureAudit]
|
||||||
# @PRE: effective template params and dataset execution reference are available.
|
# @PRE: effective template params and dataset execution reference are available.
|
||||||
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
|
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
# Test Report: Superset Preview Pipeline Validation
|
||||||
|
|
||||||
|
**Date**: 2026-03-17
|
||||||
|
**Executed by**: Tester Agent (Kilo Code)
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
Validated the Superset preview pipeline fix set around:
|
||||||
|
|
||||||
|
- preview compilation using the real [`/chart/data`](backend/src/core/superset_client.py) path
|
||||||
|
- prevention of false dashboard-not-found semantics for non-dashboard preview 404 responses
|
||||||
|
- preservation of the current API response contract in [`backend/src/api/routes/__tests__/test_dataset_review_api.py`](backend/src/api/routes/__tests__/test_dataset_review_api.py)
|
||||||
|
|
||||||
|
## Files Changed
|
||||||
|
|
||||||
|
- `backend/src/api/routes/__tests__/test_dataset_review_api.py`
|
||||||
|
- `backend/src/core/__tests__/test_superset_preview_pipeline.py`
|
||||||
|
|
||||||
|
## Coverage Added
|
||||||
|
|
||||||
|
### API-level coverage
|
||||||
|
|
||||||
|
In `backend/src/api/routes/__tests__/test_dataset_review_api.py`:
|
||||||
|
|
||||||
|
- verifies preview endpoint still returns the existing preview DTO contract when preview compilation fails
|
||||||
|
- verifies failed preview payload remains generic for non-dashboard 404 behavior
|
||||||
|
- verifies error details mention the upstream resource path and do **not** regress into false dashboard-not-found wording
|
||||||
|
|
||||||
|
### Core preview/network coverage
|
||||||
|
|
||||||
|
In `backend/src/core/__tests__/test_superset_preview_pipeline.py`:
|
||||||
|
|
||||||
|
- verifies [`SupersetClient.compile_dataset_preview`](backend/src/core/superset_client.py) calls the explicit chart-data endpoint
|
||||||
|
- verifies compiled SQL is extracted from `result[].query`
|
||||||
|
- verifies sync [`APIClient`](backend/src/core/utils/network.py) keeps non-dashboard 404s as generic [`SupersetAPIError`](backend/src/core/utils/network.py)
|
||||||
|
- verifies sync [`APIClient`](backend/src/core/utils/network.py) still translates dashboard 404s into [`DashboardNotFoundError`](backend/src/core/utils/network.py)
|
||||||
|
- verifies async [`AsyncAPIClient`](backend/src/core/utils/async_network.py) keeps non-dashboard 404s as generic [`SupersetAPIError`](backend/src/core/utils/network.py)
|
||||||
|
- verifies async [`AsyncAPIClient`](backend/src/core/utils/async_network.py) still translates dashboard 404s into [`DashboardNotFoundError`](backend/src/core/utils/network.py)
|
||||||
|
|
||||||
|
## Semantic Audit
|
||||||
|
|
||||||
|
Ran workspace semantic audit through `axiom-core`.
|
||||||
|
|
||||||
|
### Result
|
||||||
|
|
||||||
|
- Workspace warnings: **1341**
|
||||||
|
- Audit status for this task: pre-existing global semantic debt present across workspace
|
||||||
|
- Targeted change set proceeded because no blocking semantic failure was isolated to the modified test scope
|
||||||
|
|
||||||
|
## Test Commands Executed
|
||||||
|
|
||||||
|
1. Initial direct targeted run:
|
||||||
|
- `cd backend && .venv/bin/python3 -m pytest src/api/routes/__tests__/test_dataset_review_api.py src/core/__tests__/test_superset_preview_pipeline.py -q`
|
||||||
|
- **Failed during collection**
|
||||||
|
- Cause: app import triggered DB init against local PostgreSQL
|
||||||
|
|
||||||
|
2. Isolated targeted run with SQLite-backed env overrides:
|
||||||
|
- `cd backend && DATABASE_URL=sqlite:///./test_app.db TASKS_DATABASE_URL=sqlite:///./test_tasks.db AUTH_DATABASE_URL=sqlite:///./test_auth.db .venv/bin/python3 -m pytest src/api/routes/__tests__/test_dataset_review_api.py src/core/__tests__/test_superset_preview_pipeline.py -q`
|
||||||
|
- **Failed**
|
||||||
|
- Cause: two then one mismatched assertion(s) against actual [`DashboardNotFoundError`](backend/src/core/utils/network.py) message format
|
||||||
|
|
||||||
|
3. Final rerun with same SQLite overrides:
|
||||||
|
- `cd backend && DATABASE_URL=sqlite:///./test_app.db TASKS_DATABASE_URL=sqlite:///./test_tasks.db AUTH_DATABASE_URL=sqlite:///./test_auth.db .venv/bin/python3 -m pytest src/api/routes/__tests__/test_dataset_review_api.py src/core/__tests__/test_superset_preview_pipeline.py -q`
|
||||||
|
- **Passed**
|
||||||
|
|
||||||
|
## Final Test Result
|
||||||
|
|
||||||
|
- Total targeted tests passed: **20**
|
||||||
|
- Failed: **0**
|
||||||
|
- Status: **PASS**
|
||||||
|
|
||||||
|
## Notes / Remaining Limitations
|
||||||
|
|
||||||
|
- The targeted API tests require temporary DB URL overrides because importing [`src.app`](backend/src/app.py) initializes the database eagerly through [`init_db()`](backend/src/core/database.py).
|
||||||
|
- The semantic audit still reports substantial unrelated pre-existing workspace warnings.
|
||||||
|
- This change set does not add direct unit tests for [`SupersetCompilationAdapter.compile_preview()`](backend/src/core/utils/superset_compilation_adapter.py); coverage is achieved indirectly through API failure-contract assertions plus direct client/network tests.
|
||||||
Reference in New Issue
Block a user