feat(027): Final Phase T038-T043 implementation
- T038: SessionEvent logger and persistence logic - Added SessionEventLogger service with explicit audit event persistence - Added SessionEvent model with events relationship on DatasetReviewSession - Integrated event logging into orchestrator flows and API mutation endpoints - T039: Semantic source version propagation - Added source_version column to SemanticFieldEntry - Added propagate_source_version_update() to SemanticResolver - Preserves locked/manual field invariants during propagation - T040: Batch approval API and UI actions - Added batch semantic approval endpoint (/fields/semantic/approve-batch) - Added batch mapping approval endpoint (/mappings/approve-batch) - Added batch approval actions to SemanticLayerReview and ExecutionMappingReview components - Aligned batch semantics with single-item approval contracts - T041: Superset compatibility matrix tests - Added test_superset_matrix.py with preview and SQL Lab fallback coverage - Tests verify client method preference and matrix fallback behavior - T042: RBAC audit sweep on session-mutation endpoints - Added _require_owner_mutation_scope() helper - Applied owner guards to update_session, delete_session, and all mutation endpoints - Ensured no bypass of existing permission checks - T043: i18n coverage for dataset-review UI - Added workspace state labels (empty/importing/review) to en.json and ru.json - Added batch action labels for semantics and mappings - Fixed workspace state comparison to lowercase strings - Removed hardcoded workspace state display strings Signed-off-by: Implementation Specialist <impl@ss-tools>
This commit is contained in:
406
backend/src/core/utils/superset_compilation_adapter.py
Normal file
406
backend/src/core/utils/superset_compilation_adapter.py
Normal file
@@ -0,0 +1,406 @@
|
||||
# [DEF:SupersetCompilationAdapter:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: dataset_review, superset, compilation_preview, sql_lab_launch, execution_truth
|
||||
# @PURPOSE: Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context.
|
||||
# @LAYER: Infra
|
||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
||||
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
|
||||
# @RELATION: [DEPENDS_ON] ->[DatasetRunContext]
|
||||
# @PRE: effective template params and dataset execution reference are available.
|
||||
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
|
||||
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
|
||||
# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# [DEF:SupersetCompilationAdapter.imports:Block]
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from src.core.config_models import Environment
|
||||
from src.core.logger import belief_scope, logger
|
||||
from src.core.superset_client import SupersetClient
|
||||
from src.models.dataset_review import CompiledPreview, PreviewStatus
|
||||
# [/DEF:SupersetCompilationAdapter.imports:Block]
|
||||
|
||||
|
||||
# [DEF:PreviewCompilationPayload:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Typed preview payload for Superset-side compilation.
|
||||
@dataclass(frozen=True)
|
||||
class PreviewCompilationPayload:
|
||||
session_id: str
|
||||
dataset_id: int
|
||||
preview_fingerprint: str
|
||||
template_params: Dict[str, Any]
|
||||
effective_filters: List[Dict[str, Any]]
|
||||
# [/DEF:PreviewCompilationPayload:Class]
|
||||
|
||||
|
||||
# [DEF:SqlLabLaunchPayload:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Typed SQL Lab payload for audited launch handoff.
|
||||
@dataclass(frozen=True)
|
||||
class SqlLabLaunchPayload:
|
||||
session_id: str
|
||||
dataset_id: int
|
||||
preview_id: str
|
||||
compiled_sql: str
|
||||
template_params: Dict[str, Any]
|
||||
# [/DEF:SqlLabLaunchPayload:Class]
|
||||
|
||||
|
||||
# [DEF:SupersetCompilationAdapter:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
|
||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
||||
# @PRE: environment is configured and Superset is reachable for the target session.
|
||||
# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
|
||||
# @SIDE_EFFECT: issues network requests to Superset API surfaces.
|
||||
class SupersetCompilationAdapter:
|
||||
# [DEF:SupersetCompilationAdapter.__init__:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Bind adapter to one Superset environment and client instance.
|
||||
def __init__(self, environment: Environment, client: Optional[SupersetClient] = None) -> None:
|
||||
self.environment = environment
|
||||
self.client = client or SupersetClient(environment)
|
||||
# [/DEF:SupersetCompilationAdapter.__init__:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter.compile_preview:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Request Superset-side compiled SQL preview for the current effective inputs.
|
||||
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_superset_preview]
|
||||
# @PRE: dataset_id and effective inputs are available for the current session.
|
||||
# @POST: returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics.
|
||||
# @SIDE_EFFECT: performs upstream preview requests.
|
||||
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[CompiledPreview]
|
||||
def compile_preview(self, payload: PreviewCompilationPayload) -> CompiledPreview:
|
||||
with belief_scope("SupersetCompilationAdapter.compile_preview"):
|
||||
if payload.dataset_id <= 0:
|
||||
logger.explore(
|
||||
"Preview compilation rejected because dataset identifier is invalid",
|
||||
extra={"dataset_id": payload.dataset_id, "session_id": payload.session_id},
|
||||
)
|
||||
raise ValueError("dataset_id must be a positive integer")
|
||||
|
||||
logger.reason(
|
||||
"Requesting Superset-generated SQL preview",
|
||||
extra={
|
||||
"session_id": payload.session_id,
|
||||
"dataset_id": payload.dataset_id,
|
||||
"template_param_count": len(payload.template_params),
|
||||
"filter_count": len(payload.effective_filters),
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
preview_result = self._request_superset_preview(payload)
|
||||
except Exception as exc:
|
||||
logger.explore(
|
||||
"Superset preview compilation failed with explicit upstream error",
|
||||
extra={
|
||||
"session_id": payload.session_id,
|
||||
"dataset_id": payload.dataset_id,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
return CompiledPreview(
|
||||
session_id=payload.session_id,
|
||||
preview_status=PreviewStatus.FAILED,
|
||||
compiled_sql=None,
|
||||
preview_fingerprint=payload.preview_fingerprint,
|
||||
compiled_by="superset",
|
||||
error_code="superset_preview_failed",
|
||||
error_details=str(exc),
|
||||
compiled_at=None,
|
||||
)
|
||||
|
||||
compiled_sql = str(preview_result.get("compiled_sql") or "").strip()
|
||||
if not compiled_sql:
|
||||
logger.explore(
|
||||
"Superset preview response did not include compiled SQL",
|
||||
extra={
|
||||
"session_id": payload.session_id,
|
||||
"dataset_id": payload.dataset_id,
|
||||
"response_keys": sorted(preview_result.keys()),
|
||||
},
|
||||
)
|
||||
return CompiledPreview(
|
||||
session_id=payload.session_id,
|
||||
preview_status=PreviewStatus.FAILED,
|
||||
compiled_sql=None,
|
||||
preview_fingerprint=payload.preview_fingerprint,
|
||||
compiled_by="superset",
|
||||
error_code="superset_preview_empty",
|
||||
error_details="Superset preview response did not include compiled SQL",
|
||||
compiled_at=None,
|
||||
)
|
||||
|
||||
preview = CompiledPreview(
|
||||
session_id=payload.session_id,
|
||||
preview_status=PreviewStatus.READY,
|
||||
compiled_sql=compiled_sql,
|
||||
preview_fingerprint=payload.preview_fingerprint,
|
||||
compiled_by="superset",
|
||||
error_code=None,
|
||||
error_details=None,
|
||||
compiled_at=datetime.utcnow(),
|
||||
)
|
||||
logger.reflect(
|
||||
"Superset-generated SQL preview captured successfully",
|
||||
extra={
|
||||
"session_id": payload.session_id,
|
||||
"dataset_id": payload.dataset_id,
|
||||
"compiled_sql_length": len(compiled_sql),
|
||||
},
|
||||
)
|
||||
return preview
|
||||
# [/DEF:SupersetCompilationAdapter.compile_preview:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter.mark_preview_stale:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Invalidate previous preview after mapping or value changes.
|
||||
# @PRE: preview is a persisted preview artifact or current in-memory snapshot.
|
||||
# @POST: preview status becomes stale without fabricating a replacement artifact.
|
||||
def mark_preview_stale(self, preview: CompiledPreview) -> CompiledPreview:
|
||||
preview.preview_status = PreviewStatus.STALE
|
||||
return preview
|
||||
# [/DEF:SupersetCompilationAdapter.mark_preview_stale:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter.create_sql_lab_session:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Create the canonical audited execution session after all launch gates pass.
|
||||
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_sql_lab_session]
|
||||
# @PRE: compiled_sql is Superset-originated and launch gates are already satisfied.
|
||||
# @POST: returns one canonical SQL Lab session reference from Superset.
|
||||
# @SIDE_EFFECT: performs upstream SQL Lab execution/session creation.
|
||||
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[str]
|
||||
def create_sql_lab_session(self, payload: SqlLabLaunchPayload) -> str:
|
||||
with belief_scope("SupersetCompilationAdapter.create_sql_lab_session"):
|
||||
compiled_sql = str(payload.compiled_sql or "").strip()
|
||||
if not compiled_sql:
|
||||
logger.explore(
|
||||
"SQL Lab launch rejected because compiled SQL is empty",
|
||||
extra={"session_id": payload.session_id, "preview_id": payload.preview_id},
|
||||
)
|
||||
raise ValueError("compiled_sql must be non-empty")
|
||||
|
||||
logger.reason(
|
||||
"Creating SQL Lab execution session from Superset-originated preview",
|
||||
extra={
|
||||
"session_id": payload.session_id,
|
||||
"dataset_id": payload.dataset_id,
|
||||
"preview_id": payload.preview_id,
|
||||
},
|
||||
)
|
||||
result = self._request_sql_lab_session(payload)
|
||||
sql_lab_session_ref = str(
|
||||
result.get("sql_lab_session_ref")
|
||||
or result.get("query_id")
|
||||
or result.get("id")
|
||||
or result.get("result", {}).get("id")
|
||||
or ""
|
||||
).strip()
|
||||
if not sql_lab_session_ref:
|
||||
logger.explore(
|
||||
"Superset SQL Lab launch response did not include a stable session reference",
|
||||
extra={"session_id": payload.session_id, "preview_id": payload.preview_id},
|
||||
)
|
||||
raise RuntimeError("Superset SQL Lab launch response did not include a session reference")
|
||||
|
||||
logger.reflect(
|
||||
"Canonical SQL Lab session created successfully",
|
||||
extra={
|
||||
"session_id": payload.session_id,
|
||||
"preview_id": payload.preview_id,
|
||||
"sql_lab_session_ref": sql_lab_session_ref,
|
||||
},
|
||||
)
|
||||
return sql_lab_session_ref
|
||||
# [/DEF:SupersetCompilationAdapter.create_sql_lab_session:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter._request_superset_preview:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Probe supported Superset preview surfaces and return the first explicit compilation response.
|
||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
||||
# @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
|
||||
# @POST: returns the first upstream response that exposes compiled SQL without fabricating local SQL.
|
||||
# @SIDE_EFFECT: issues one or more Superset preview requests until a supported surface responds.
|
||||
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
|
||||
def _request_superset_preview(self, payload: PreviewCompilationPayload) -> Dict[str, Any]:
|
||||
request_payload = {
|
||||
"dataset_id": payload.dataset_id,
|
||||
"template_params": payload.template_params,
|
||||
"effective_filters": payload.effective_filters,
|
||||
"session_id": payload.session_id,
|
||||
}
|
||||
candidate_calls = self._build_preview_call_candidates(payload.dataset_id, request_payload)
|
||||
errors: List[str] = []
|
||||
|
||||
for candidate in candidate_calls:
|
||||
call_kind = candidate["kind"]
|
||||
target = candidate["target"]
|
||||
try:
|
||||
logger.reason(
|
||||
"Attempting Superset preview compilation candidate",
|
||||
extra={"kind": call_kind, "target": target},
|
||||
)
|
||||
if call_kind == "client_method":
|
||||
method = getattr(self.client, target)
|
||||
response = method(request_payload)
|
||||
else:
|
||||
response = self.client.network.request(
|
||||
method=candidate["http_method"],
|
||||
endpoint=target,
|
||||
data=candidate["data"],
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
normalized = self._normalize_preview_response(response)
|
||||
if normalized is not None:
|
||||
return normalized
|
||||
except Exception as exc:
|
||||
errors.append(f"{call_kind}:{target}:{exc}")
|
||||
logger.explore(
|
||||
"Superset preview compilation candidate failed",
|
||||
extra={"kind": call_kind, "target": target, "error": str(exc)},
|
||||
)
|
||||
|
||||
raise RuntimeError("; ".join(errors) or "No Superset preview surface accepted the request")
|
||||
# [/DEF:SupersetCompilationAdapter._request_superset_preview:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response.
|
||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
||||
# @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
|
||||
# @POST: returns the first successful SQL Lab execution response from Superset.
|
||||
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
|
||||
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
|
||||
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]:
|
||||
dataset_raw = self.client.get_dataset(payload.dataset_id)
|
||||
dataset_record = dataset_raw.get("result", dataset_raw) if isinstance(dataset_raw, dict) else {}
|
||||
database_id = dataset_record.get("database", {}).get("id") if isinstance(dataset_record.get("database"), dict) else dataset_record.get("database_id")
|
||||
if database_id is None:
|
||||
raise RuntimeError("Superset dataset does not expose a database identifier for SQL Lab launch")
|
||||
|
||||
request_payload = {
|
||||
"database_id": database_id,
|
||||
"sql": payload.compiled_sql,
|
||||
"templateParams": payload.template_params,
|
||||
"schema": dataset_record.get("schema"),
|
||||
"client_id": payload.preview_id,
|
||||
}
|
||||
candidate_calls = [
|
||||
{"kind": "network", "target": "/sqllab/execute/", "http_method": "POST"},
|
||||
{"kind": "network", "target": "/sql_lab/execute/", "http_method": "POST"},
|
||||
]
|
||||
errors: List[str] = []
|
||||
|
||||
for candidate in candidate_calls:
|
||||
try:
|
||||
response = self.client.network.request(
|
||||
method=candidate["http_method"],
|
||||
endpoint=candidate["target"],
|
||||
data=self._dump_json(request_payload),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
if isinstance(response, dict) and response:
|
||||
return response
|
||||
except Exception as exc:
|
||||
errors.append(f"{candidate['target']}:{exc}")
|
||||
logger.explore(
|
||||
"Superset SQL Lab candidate failed",
|
||||
extra={"target": candidate["target"], "error": str(exc)},
|
||||
)
|
||||
|
||||
raise RuntimeError("; ".join(errors) or "No Superset SQL Lab surface accepted the request")
|
||||
# [/DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter._build_preview_call_candidates:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Assemble preview candidate call shapes in priority order.
|
||||
def _build_preview_call_candidates(
|
||||
self,
|
||||
dataset_id: int,
|
||||
request_payload: Dict[str, Any],
|
||||
) -> List[Dict[str, Any]]:
|
||||
candidates: List[Dict[str, Any]] = []
|
||||
for method_name in (
|
||||
"compile_sql_preview",
|
||||
"compile_preview",
|
||||
"get_compiled_sql_preview",
|
||||
):
|
||||
if hasattr(self.client, method_name):
|
||||
candidates.append({"kind": "client_method", "target": method_name})
|
||||
|
||||
encoded_payload = self._dump_json(request_payload)
|
||||
candidates.extend(
|
||||
[
|
||||
{
|
||||
"kind": "network",
|
||||
"target": f"/dataset/{dataset_id}/preview",
|
||||
"http_method": "POST",
|
||||
"data": encoded_payload,
|
||||
},
|
||||
{
|
||||
"kind": "network",
|
||||
"target": f"/dataset/{dataset_id}/sql",
|
||||
"http_method": "POST",
|
||||
"data": encoded_payload,
|
||||
},
|
||||
{
|
||||
"kind": "network",
|
||||
"target": "/sqllab/format_sql/",
|
||||
"http_method": "POST",
|
||||
"data": encoded_payload,
|
||||
},
|
||||
]
|
||||
)
|
||||
return candidates
|
||||
# [/DEF:SupersetCompilationAdapter._build_preview_call_candidates:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
|
||||
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
|
||||
def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]:
|
||||
if not isinstance(response, dict):
|
||||
return None
|
||||
|
||||
compiled_sql_candidates = [
|
||||
response.get("compiled_sql"),
|
||||
response.get("sql"),
|
||||
response.get("query"),
|
||||
]
|
||||
result_payload = response.get("result")
|
||||
if isinstance(result_payload, dict):
|
||||
compiled_sql_candidates.extend(
|
||||
[
|
||||
result_payload.get("compiled_sql"),
|
||||
result_payload.get("sql"),
|
||||
result_payload.get("query"),
|
||||
]
|
||||
)
|
||||
|
||||
for candidate in compiled_sql_candidates:
|
||||
compiled_sql = str(candidate or "").strip()
|
||||
if compiled_sql:
|
||||
return {
|
||||
"compiled_sql": compiled_sql,
|
||||
"raw_response": response,
|
||||
}
|
||||
return None
|
||||
# [/DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
|
||||
|
||||
# [DEF:SupersetCompilationAdapter._dump_json:Function]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Serialize Superset request payload deterministically for network transport.
|
||||
def _dump_json(self, payload: Dict[str, Any]) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(payload, sort_keys=True, default=str)
|
||||
# [/DEF:SupersetCompilationAdapter._dump_json:Function]
|
||||
# [/DEF:SupersetCompilationAdapter:Class]
|
||||
|
||||
# [/DEF:SupersetCompilationAdapter:Module]
|
||||
@@ -15,8 +15,9 @@ from __future__ import annotations
|
||||
|
||||
# [DEF:SupersetContextExtractor.imports:Block]
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from src.core.config_models import Environment
|
||||
@@ -204,17 +205,224 @@ class SupersetContextExtractor:
|
||||
# [/DEF:SupersetContextExtractor.parse_superset_link:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor.recover_imported_filters:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Build imported filter entries from URL state and Superset-side saved context.
|
||||
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
|
||||
# @PRE: parsed_context comes from a successful Superset link parse for one environment.
|
||||
# @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
|
||||
# @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment.
|
||||
# @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
|
||||
def recover_imported_filters(self, parsed_context: SupersetParsedContext) -> List[Dict[str, Any]]:
|
||||
return list(parsed_context.imported_filters)
|
||||
with belief_scope("SupersetContextExtractor.recover_imported_filters"):
|
||||
recovered_filters: List[Dict[str, Any]] = []
|
||||
seen_filter_keys: Set[str] = set()
|
||||
|
||||
for item in parsed_context.imported_filters:
|
||||
normalized = self._normalize_imported_filter_payload(
|
||||
item,
|
||||
default_source="superset_url",
|
||||
default_note="Recovered from Superset URL state",
|
||||
)
|
||||
filter_key = normalized["filter_name"].strip().lower()
|
||||
if filter_key in seen_filter_keys:
|
||||
continue
|
||||
seen_filter_keys.add(filter_key)
|
||||
recovered_filters.append(normalized)
|
||||
|
||||
if parsed_context.dashboard_id is None:
|
||||
logger.reflect(
|
||||
"Imported filter recovery completed without dashboard enrichment",
|
||||
extra={
|
||||
"dashboard_id": None,
|
||||
"filter_count": len(recovered_filters),
|
||||
"partial_recovery": parsed_context.partial_recovery,
|
||||
},
|
||||
)
|
||||
return recovered_filters
|
||||
|
||||
try:
|
||||
dashboard_payload = self.client.get_dashboard(parsed_context.dashboard_id)
|
||||
dashboard_record = (
|
||||
dashboard_payload.get("result", dashboard_payload)
|
||||
if isinstance(dashboard_payload, dict)
|
||||
else {}
|
||||
)
|
||||
json_metadata = dashboard_record.get("json_metadata")
|
||||
if isinstance(json_metadata, str) and json_metadata.strip():
|
||||
json_metadata = json.loads(json_metadata)
|
||||
if not isinstance(json_metadata, dict):
|
||||
json_metadata = {}
|
||||
|
||||
native_filter_configuration = json_metadata.get("native_filter_configuration") or []
|
||||
default_filters = json_metadata.get("default_filters") or {}
|
||||
if isinstance(default_filters, str) and default_filters.strip():
|
||||
try:
|
||||
default_filters = json.loads(default_filters)
|
||||
except Exception:
|
||||
logger.explore(
|
||||
"Superset default_filters payload was not valid JSON",
|
||||
extra={"dashboard_id": parsed_context.dashboard_id},
|
||||
)
|
||||
default_filters = {}
|
||||
|
||||
for item in native_filter_configuration:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
filter_name = str(
|
||||
item.get("name")
|
||||
or item.get("filter_name")
|
||||
or item.get("column")
|
||||
or ""
|
||||
).strip()
|
||||
if not filter_name:
|
||||
continue
|
||||
|
||||
filter_key = filter_name.lower()
|
||||
if filter_key in seen_filter_keys:
|
||||
continue
|
||||
|
||||
default_value = None
|
||||
if isinstance(default_filters, dict):
|
||||
default_value = default_filters.get(filter_name)
|
||||
|
||||
saved_filter = self._normalize_imported_filter_payload(
|
||||
{
|
||||
"filter_name": filter_name,
|
||||
"display_name": item.get("label") or item.get("name"),
|
||||
"raw_value": default_value,
|
||||
"source": "superset_native",
|
||||
"recovery_status": "recovered" if default_value is not None else "partial",
|
||||
"requires_confirmation": default_value is None,
|
||||
"notes": "Recovered from Superset dashboard native filter configuration",
|
||||
},
|
||||
default_source="superset_native",
|
||||
default_note="Recovered from Superset dashboard native filter configuration",
|
||||
)
|
||||
seen_filter_keys.add(filter_key)
|
||||
recovered_filters.append(saved_filter)
|
||||
|
||||
logger.reflect(
|
||||
"Imported filter recovery completed with dashboard enrichment",
|
||||
extra={
|
||||
"dashboard_id": parsed_context.dashboard_id,
|
||||
"filter_count": len(recovered_filters),
|
||||
"partial_entries": len(
|
||||
[
|
||||
item
|
||||
for item in recovered_filters
|
||||
if item["recovery_status"] == "partial"
|
||||
]
|
||||
),
|
||||
},
|
||||
)
|
||||
return recovered_filters
|
||||
except Exception as exc:
|
||||
logger.explore(
|
||||
"Dashboard native filter enrichment failed; preserving partial imported filters",
|
||||
extra={
|
||||
"dashboard_id": parsed_context.dashboard_id,
|
||||
"error": str(exc),
|
||||
"filter_count": len(recovered_filters),
|
||||
},
|
||||
)
|
||||
if not recovered_filters:
|
||||
recovered_filters.append(
|
||||
self._normalize_imported_filter_payload(
|
||||
{
|
||||
"filter_name": f"dashboard_{parsed_context.dashboard_id}_filters",
|
||||
"display_name": "Dashboard native filters",
|
||||
"raw_value": None,
|
||||
"source": "superset_native",
|
||||
"recovery_status": "partial",
|
||||
"requires_confirmation": True,
|
||||
"notes": "Superset dashboard filter configuration could not be recovered fully",
|
||||
},
|
||||
default_source="superset_native",
|
||||
default_note="Superset dashboard filter configuration could not be recovered fully",
|
||||
)
|
||||
)
|
||||
return recovered_filters
|
||||
# [/DEF:SupersetContextExtractor.recover_imported_filters:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor.discover_template_variables:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields.
|
||||
# @RELATION: [DEPENDS_ON] ->[TemplateVariable]
|
||||
# @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
|
||||
# @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
|
||||
# @SIDE_EFFECT: none.
|
||||
# @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
|
||||
def discover_template_variables(self, dataset_payload: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
return []
|
||||
with belief_scope("SupersetContextExtractor.discover_template_variables"):
|
||||
discovered: List[Dict[str, Any]] = []
|
||||
seen_variable_names: Set[str] = set()
|
||||
|
||||
for expression_source in self._collect_query_bearing_expressions(dataset_payload):
|
||||
for filter_match in re.finditer(
|
||||
r"filter_values\(\s*['\"]([^'\"]+)['\"]\s*\)",
|
||||
expression_source,
|
||||
flags=re.IGNORECASE,
|
||||
):
|
||||
variable_name = str(filter_match.group(1) or "").strip()
|
||||
if not variable_name:
|
||||
continue
|
||||
self._append_template_variable(
|
||||
discovered=discovered,
|
||||
seen_variable_names=seen_variable_names,
|
||||
variable_name=variable_name,
|
||||
expression_source=expression_source,
|
||||
variable_kind="native_filter",
|
||||
is_required=True,
|
||||
default_value=None,
|
||||
)
|
||||
|
||||
for url_param_match in re.finditer(
|
||||
r"url_param\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*([^)]+))?\)",
|
||||
expression_source,
|
||||
flags=re.IGNORECASE,
|
||||
):
|
||||
variable_name = str(url_param_match.group(1) or "").strip()
|
||||
if not variable_name:
|
||||
continue
|
||||
default_literal = url_param_match.group(2)
|
||||
self._append_template_variable(
|
||||
discovered=discovered,
|
||||
seen_variable_names=seen_variable_names,
|
||||
variable_name=variable_name,
|
||||
expression_source=expression_source,
|
||||
variable_kind="parameter",
|
||||
is_required=default_literal is None,
|
||||
default_value=self._normalize_default_literal(default_literal),
|
||||
)
|
||||
|
||||
for jinja_match in re.finditer(r"\{\{\s*(.*?)\s*\}\}", expression_source, flags=re.DOTALL):
|
||||
expression = str(jinja_match.group(1) or "").strip()
|
||||
if not expression:
|
||||
continue
|
||||
if any(token in expression for token in ("filter_values(", "url_param(", "get_filters(")):
|
||||
continue
|
||||
variable_name = self._extract_primary_jinja_identifier(expression)
|
||||
if not variable_name:
|
||||
continue
|
||||
self._append_template_variable(
|
||||
discovered=discovered,
|
||||
seen_variable_names=seen_variable_names,
|
||||
variable_name=variable_name,
|
||||
expression_source=expression_source,
|
||||
variable_kind="derived" if "." in expression or "|" in expression else "parameter",
|
||||
is_required=True,
|
||||
default_value=None,
|
||||
)
|
||||
|
||||
logger.reflect(
|
||||
"Template variable discovery completed deterministically",
|
||||
extra={
|
||||
"dataset_id": dataset_payload.get("id"),
|
||||
"variable_count": len(discovered),
|
||||
"variable_names": [item["variable_name"] for item in discovered],
|
||||
},
|
||||
)
|
||||
return discovered
|
||||
# [/DEF:SupersetContextExtractor.discover_template_variables:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor.build_recovery_summary:Function]
|
||||
@@ -329,6 +537,151 @@ class SupersetContextExtractor:
|
||||
|
||||
return imported_filters
|
||||
# [/DEF:SupersetContextExtractor._extract_imported_filters:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor._normalize_imported_filter_payload:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state.
|
||||
def _normalize_imported_filter_payload(
|
||||
self,
|
||||
payload: Dict[str, Any],
|
||||
default_source: str,
|
||||
default_note: str,
|
||||
) -> Dict[str, Any]:
|
||||
raw_value = payload.get("raw_value")
|
||||
if "raw_value" not in payload and "value" in payload:
|
||||
raw_value = payload.get("value")
|
||||
|
||||
recovery_status = str(
|
||||
payload.get("recovery_status")
|
||||
or ("recovered" if raw_value is not None else "partial")
|
||||
).strip().lower()
|
||||
requires_confirmation = bool(
|
||||
payload.get("requires_confirmation", raw_value is None or recovery_status != "recovered")
|
||||
)
|
||||
return {
|
||||
"filter_name": str(payload.get("filter_name") or "unresolved_filter").strip(),
|
||||
"display_name": payload.get("display_name"),
|
||||
"raw_value": raw_value,
|
||||
"normalized_value": payload.get("normalized_value"),
|
||||
"source": str(payload.get("source") or default_source),
|
||||
"confidence_state": "imported" if raw_value is not None else "unresolved",
|
||||
"requires_confirmation": requires_confirmation,
|
||||
"recovery_status": recovery_status,
|
||||
"notes": str(payload.get("notes") or default_note),
|
||||
}
|
||||
# [/DEF:SupersetContextExtractor._normalize_imported_filter_payload:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor._collect_query_bearing_expressions:Function]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
|
||||
# @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor.discover_template_variables]
|
||||
def _collect_query_bearing_expressions(self, dataset_payload: Dict[str, Any]) -> List[str]:
|
||||
expressions: List[str] = []
|
||||
|
||||
def append_expression(candidate: Any) -> None:
|
||||
if not isinstance(candidate, str):
|
||||
return
|
||||
normalized = candidate.strip()
|
||||
if normalized:
|
||||
expressions.append(normalized)
|
||||
|
||||
append_expression(dataset_payload.get("sql"))
|
||||
append_expression(dataset_payload.get("query"))
|
||||
append_expression(dataset_payload.get("template_sql"))
|
||||
|
||||
metrics_payload = dataset_payload.get("metrics") or []
|
||||
if isinstance(metrics_payload, list):
|
||||
for metric in metrics_payload:
|
||||
if isinstance(metric, str):
|
||||
append_expression(metric)
|
||||
continue
|
||||
if not isinstance(metric, dict):
|
||||
continue
|
||||
append_expression(metric.get("expression"))
|
||||
append_expression(metric.get("sqlExpression"))
|
||||
append_expression(metric.get("metric_name"))
|
||||
|
||||
columns_payload = dataset_payload.get("columns") or []
|
||||
if isinstance(columns_payload, list):
|
||||
for column in columns_payload:
|
||||
if not isinstance(column, dict):
|
||||
continue
|
||||
append_expression(column.get("sqlExpression"))
|
||||
append_expression(column.get("expression"))
|
||||
|
||||
return expressions
|
||||
# [/DEF:SupersetContextExtractor._collect_query_bearing_expressions:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor._append_template_variable:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Append one deduplicated template-variable descriptor.
|
||||
def _append_template_variable(
|
||||
self,
|
||||
discovered: List[Dict[str, Any]],
|
||||
seen_variable_names: Set[str],
|
||||
variable_name: str,
|
||||
expression_source: str,
|
||||
variable_kind: str,
|
||||
is_required: bool,
|
||||
default_value: Any,
|
||||
) -> None:
|
||||
normalized_name = str(variable_name or "").strip()
|
||||
if not normalized_name:
|
||||
return
|
||||
seen_key = normalized_name.lower()
|
||||
if seen_key in seen_variable_names:
|
||||
return
|
||||
seen_variable_names.add(seen_key)
|
||||
discovered.append(
|
||||
{
|
||||
"variable_name": normalized_name,
|
||||
"expression_source": expression_source,
|
||||
"variable_kind": variable_kind,
|
||||
"is_required": is_required,
|
||||
"default_value": default_value,
|
||||
"mapping_status": "unmapped",
|
||||
}
|
||||
)
|
||||
# [/DEF:SupersetContextExtractor._append_template_variable:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor._extract_primary_jinja_identifier:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it.
|
||||
def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]:
|
||||
matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip())
|
||||
if matched is None:
|
||||
return None
|
||||
candidate = matched.group(1)
|
||||
if candidate in {"if", "else", "for", "set", "True", "False", "none", "None"}:
|
||||
return None
|
||||
return candidate
|
||||
# [/DEF:SupersetContextExtractor._extract_primary_jinja_identifier:Function]
|
||||
|
||||
# [DEF:SupersetContextExtractor._normalize_default_literal:Function]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values.
|
||||
def _normalize_default_literal(self, literal: Optional[str]) -> Any:
|
||||
normalized_literal = str(literal or "").strip()
|
||||
if not normalized_literal:
|
||||
return None
|
||||
if (
|
||||
(normalized_literal.startswith("'") and normalized_literal.endswith("'"))
|
||||
or (normalized_literal.startswith('"') and normalized_literal.endswith('"'))
|
||||
):
|
||||
return normalized_literal[1:-1]
|
||||
lowered = normalized_literal.lower()
|
||||
if lowered in {"true", "false"}:
|
||||
return lowered == "true"
|
||||
if lowered in {"null", "none"}:
|
||||
return None
|
||||
try:
|
||||
return int(normalized_literal)
|
||||
except ValueError:
|
||||
try:
|
||||
return float(normalized_literal)
|
||||
except ValueError:
|
||||
return normalized_literal
|
||||
# [/DEF:SupersetContextExtractor._normalize_default_literal:Function]
|
||||
# [/DEF:SupersetContextExtractor:Class]
|
||||
|
||||
# [/DEF:SupersetContextExtractor:Module]
|
||||
Reference in New Issue
Block a user