chore(semantic): checkpoint remediation progress
This commit is contained in:
@@ -4,9 +4,9 @@
|
||||
# @SEMANTICS: api, datasets, resources, hub
|
||||
# @PURPOSE: API endpoints for the Dataset Hub - listing datasets with mapping progress
|
||||
# @LAYER: API
|
||||
# @RELATION: DEPENDS_ON -> backend.src.dependencies
|
||||
# @RELATION: DEPENDS_ON -> backend.src.services.resource_service
|
||||
# @RELATION: DEPENDS_ON -> backend.src.core.superset_client
|
||||
# @RELATION: DEPENDS_ON ->[backend.src.dependencies]
|
||||
# @RELATION: DEPENDS_ON ->[backend.src.services.resource_service]
|
||||
# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client]
|
||||
#
|
||||
# @INVARIANT: All dataset responses include last_task metadata
|
||||
|
||||
@@ -22,28 +22,39 @@ from ...core.superset_client import SupersetClient
|
||||
router = APIRouter(prefix="/api/datasets", tags=["Datasets"])
|
||||
|
||||
# [DEF:MappedFields:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: DTO for dataset mapping progress statistics
|
||||
class MappedFields(BaseModel):
|
||||
total: int
|
||||
mapped: int
|
||||
# [/DEF:MappedFields:DataClass]
|
||||
|
||||
# [DEF:LastTask:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: DTO for the most recent task associated with a dataset
|
||||
class LastTask(BaseModel):
|
||||
task_id: Optional[str] = None
|
||||
status: Optional[str] = Field(None, pattern="^RUNNING|SUCCESS|ERROR|WAITING_INPUT$")
|
||||
# [/DEF:LastTask:DataClass]
|
||||
|
||||
# [DEF:DatasetItem:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Summary DTO for a dataset in the hub listing
|
||||
class DatasetItem(BaseModel):
|
||||
id: int
|
||||
table_name: str
|
||||
schema: str
|
||||
schema_name: str = Field(..., alias="schema")
|
||||
database: str
|
||||
mapped_fields: Optional[MappedFields] = None
|
||||
last_task: Optional[LastTask] = None
|
||||
|
||||
class Config:
|
||||
allow_population_by_field_name = True
|
||||
# [/DEF:DatasetItem:DataClass]
|
||||
|
||||
# [DEF:LinkedDashboard:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: DTO for a dashboard linked to a dataset
|
||||
class LinkedDashboard(BaseModel):
|
||||
id: int
|
||||
title: str
|
||||
@@ -51,6 +62,8 @@ class LinkedDashboard(BaseModel):
|
||||
# [/DEF:LinkedDashboard:DataClass]
|
||||
|
||||
# [DEF:DatasetColumn:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: DTO for a single dataset column's metadata
|
||||
class DatasetColumn(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
@@ -61,10 +74,12 @@ class DatasetColumn(BaseModel):
|
||||
# [/DEF:DatasetColumn:DataClass]
|
||||
|
||||
# [DEF:DatasetDetailResponse:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Detailed DTO for a dataset including columns and links
|
||||
class DatasetDetailResponse(BaseModel):
|
||||
id: int
|
||||
table_name: Optional[str] = None
|
||||
schema: Optional[str] = None
|
||||
schema_name: Optional[str] = Field(None, alias="schema")
|
||||
database: str
|
||||
description: Optional[str] = None
|
||||
columns: List[DatasetColumn]
|
||||
@@ -75,9 +90,14 @@ class DatasetDetailResponse(BaseModel):
|
||||
is_sqllab_view: bool = False
|
||||
created_on: Optional[str] = None
|
||||
changed_on: Optional[str] = None
|
||||
|
||||
class Config:
|
||||
allow_population_by_field_name = True
|
||||
# [/DEF:DatasetDetailResponse:DataClass]
|
||||
|
||||
# [DEF:DatasetsResponse:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Paginated response DTO for dataset listings
|
||||
class DatasetsResponse(BaseModel):
|
||||
datasets: List[DatasetItem]
|
||||
total: int
|
||||
@@ -87,18 +107,21 @@ class DatasetsResponse(BaseModel):
|
||||
# [/DEF:DatasetsResponse:DataClass]
|
||||
|
||||
# [DEF:TaskResponse:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Response DTO containing a task ID for tracking
|
||||
class TaskResponse(BaseModel):
|
||||
task_id: str
|
||||
# [/DEF:TaskResponse:DataClass]
|
||||
|
||||
# [DEF:get_dataset_ids:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Fetch list of all dataset IDs from a specific environment (without pagination)
|
||||
# @PRE: env_id must be a valid environment ID
|
||||
# @POST: Returns a list of all dataset IDs
|
||||
# @PARAM: env_id (str) - The environment ID to fetch datasets from
|
||||
# @PARAM: search (Optional[str]) - Filter by table name
|
||||
# @RETURN: List[int] - List of dataset IDs
|
||||
# @RELATION: CALLS -> ResourceService.get_datasets_with_status
|
||||
# @RELATION: CALLS ->[backend.src.services.resource_service.ResourceService:get_datasets_with_status]
|
||||
@router.get("/ids")
|
||||
async def get_dataset_ids(
|
||||
env_id: str,
|
||||
@@ -143,6 +166,7 @@ async def get_dataset_ids(
|
||||
# [/DEF:get_dataset_ids:Function]
|
||||
|
||||
# [DEF:get_datasets:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Fetch list of datasets from a specific environment with mapping progress
|
||||
# @PRE: env_id must be a valid environment ID
|
||||
# @PRE: page must be >= 1 if provided
|
||||
@@ -154,7 +178,7 @@ async def get_dataset_ids(
|
||||
# @PARAM: page (Optional[int]) - Page number (default: 1)
|
||||
# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100)
|
||||
# @RETURN: DatasetsResponse - List of datasets with status metadata
|
||||
# @RELATION: CALLS -> ResourceService.get_datasets_with_status
|
||||
# @RELATION: CALLS ->[backend.src.services.resource_service.ResourceService:get_datasets_with_status]
|
||||
@router.get("", response_model=DatasetsResponse)
|
||||
async def get_datasets(
|
||||
env_id: str,
|
||||
@@ -222,6 +246,8 @@ async def get_datasets(
|
||||
# [/DEF:get_datasets:Function]
|
||||
|
||||
# [DEF:MapColumnsRequest:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Request DTO for initiating column mapping
|
||||
class MapColumnsRequest(BaseModel):
|
||||
env_id: str = Field(..., description="Environment ID")
|
||||
dataset_ids: List[int] = Field(..., description="List of dataset IDs to map")
|
||||
@@ -231,6 +257,7 @@ class MapColumnsRequest(BaseModel):
|
||||
# [/DEF:MapColumnsRequest:DataClass]
|
||||
|
||||
# [DEF:map_columns:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Trigger bulk column mapping for datasets
|
||||
# @PRE: User has permission plugin:mapper:execute
|
||||
# @PRE: env_id is a valid environment ID
|
||||
@@ -239,8 +266,8 @@ class MapColumnsRequest(BaseModel):
|
||||
# @POST: Task is created and queued for execution
|
||||
# @PARAM: request (MapColumnsRequest) - Mapping request with environment and dataset IDs
|
||||
# @RETURN: TaskResponse - Task ID for tracking
|
||||
# @RELATION: DISPATCHES -> MapperPlugin
|
||||
# @RELATION: CALLS -> task_manager.create_task
|
||||
# @RELATION: DISPATCHES ->[backend.src.plugins.mapper.MapperPlugin]
|
||||
# @RELATION: CALLS ->[backend.src.core.task_manager.manager.TaskManager:create_task]
|
||||
@router.post("/map-columns", response_model=TaskResponse)
|
||||
async def map_columns(
|
||||
request: MapColumnsRequest,
|
||||
@@ -292,6 +319,8 @@ async def map_columns(
|
||||
# [/DEF:map_columns:Function]
|
||||
|
||||
# [DEF:GenerateDocsRequest:DataClass]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Request DTO for initiating documentation generation
|
||||
class GenerateDocsRequest(BaseModel):
|
||||
env_id: str = Field(..., description="Environment ID")
|
||||
dataset_ids: List[int] = Field(..., description="List of dataset IDs to generate docs for")
|
||||
@@ -300,6 +329,7 @@ class GenerateDocsRequest(BaseModel):
|
||||
# [/DEF:GenerateDocsRequest:DataClass]
|
||||
|
||||
# [DEF:generate_docs:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Trigger bulk documentation generation for datasets
|
||||
# @PRE: User has permission plugin:llm_analysis:execute
|
||||
# @PRE: env_id is a valid environment ID
|
||||
@@ -308,8 +338,8 @@ class GenerateDocsRequest(BaseModel):
|
||||
# @POST: Task is created and queued for execution
|
||||
# @PARAM: request (GenerateDocsRequest) - Documentation generation request
|
||||
# @RETURN: TaskResponse - Task ID for tracking
|
||||
# @RELATION: DISPATCHES -> LLMAnalysisPlugin
|
||||
# @RELATION: CALLS -> task_manager.create_task
|
||||
# @RELATION: DISPATCHES ->[backend.src.plugins.llm_analysis.plugin.DocumentationPlugin]
|
||||
# @RELATION: CALLS ->[backend.src.core.task_manager.manager.TaskManager:create_task]
|
||||
@router.post("/generate-docs", response_model=TaskResponse)
|
||||
async def generate_docs(
|
||||
request: GenerateDocsRequest,
|
||||
@@ -355,6 +385,7 @@ async def generate_docs(
|
||||
# [/DEF:generate_docs:Function]
|
||||
|
||||
# [DEF:get_dataset_detail:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Get detailed dataset information including columns and linked dashboards
|
||||
# @PRE: env_id is a valid environment ID
|
||||
# @PRE: dataset_id is a valid dataset ID
|
||||
@@ -362,7 +393,7 @@ async def generate_docs(
|
||||
# @PARAM: env_id (str) - The environment ID
|
||||
# @PARAM: dataset_id (int) - The dataset ID
|
||||
# @RETURN: DatasetDetailResponse - Detailed dataset information
|
||||
# @RELATION: CALLS -> SupersetClient.get_dataset_detail
|
||||
# @RELATION: CALLS ->[backend.src.core.superset_client.SupersetClient:get_dataset_detail]
|
||||
@router.get("/{dataset_id}", response_model=DatasetDetailResponse)
|
||||
async def get_dataset_detail(
|
||||
env_id: str,
|
||||
|
||||
Reference in New Issue
Block a user