busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: Implement recursive storage listing and directory browsing for backups, and add a migration option to fix cross-filters.

Browse Source
This commit is contained in:
busya
2026-02-25 20:01:33 +03:00
parent 5d42a6b930
commit f9ac282596
12 changed files with 533 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
backend/src/plugins/migration.py
View File

@@ -150,6 +150,7 @@ class MigrationPlugin(PluginBase):
dashboard_regex = params.get("dashboard_regex")
replace_db_config = params.get("replace_db_config", False)
fix_cross_filters = params.get("fix_cross_filters", True)
params.get("from_db_id")
params.get("to_db_id")
@@ -217,9 +218,9 @@ class MigrationPlugin(PluginBase):
if selected_ids:
dashboards_to_migrate = [d for d in all_dashboards if d["id"] in selected_ids]
elif dashboard_regex:
regex_str = str(dashboard_regex)
regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE)
dashboards_to_migrate = [
d for d in all_dashboards if re.search(regex_str, d["dashboard_title"], re.IGNORECASE)
d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", ""))
]
else:
log.warning("No selection criteria provided (selected_ids or dashboard_regex).")
@@ -270,7 +271,14 @@ class MigrationPlugin(PluginBase):
with create_temp_file(content=exported_content, dry_run=True, suffix=".zip") as tmp_zip_path:
# Always transform to strip databases to avoid password errors
with create_temp_file(suffix=".zip", dry_run=True) as tmp_new_zip:
success = engine.transform_zip(str(tmp_zip_path), str(tmp_new_zip), db_mapping, strip_databases=False)
success = engine.transform_zip(
str(tmp_zip_path),
str(tmp_new_zip),
db_mapping,
strip_databases=False,
target_env_id=tgt_env.id if tgt_env else None,
fix_cross_filters=fix_cross_filters
)
if not success and replace_db_config:
# Signal missing mapping and wait (only if we care about mappings)
@@ -283,16 +291,23 @@ class MigrationPlugin(PluginBase):
# (Mappings would be updated in task.params by resolve_task)
db = SessionLocal()
try:
src_env = db.query(Environment).filter(Environment.name == from_env_name).first()
tgt_env = db.query(Environment).filter(Environment.name == to_env_name).first()
src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first()
tgt_env_rt = db.query(Environment).filter(Environment.name == to_env_name).first()
mappings = db.query(DatabaseMapping).filter(
DatabaseMapping.source_env_id == src_env.id,
DatabaseMapping.target_env_id == tgt_env.id
DatabaseMapping.source_env_id == src_env_rt.id,
DatabaseMapping.target_env_id == tgt_env_rt.id
).all()
db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings}
finally:
db.close()
success = engine.transform_zip(str(tmp_zip_path), str(tmp_new_zip), db_mapping, strip_databases=False)
success = engine.transform_zip(
str(tmp_zip_path),
str(tmp_new_zip),
db_mapping,
strip_databases=False,
target_env_id=tgt_env.id if tgt_env else None,
fix_cross_filters=fix_cross_filters
)
if success:
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)

40
backend/src/plugins/storage/plugin.py
View File

@@ -212,13 +212,21 @@ class StoragePlugin(PluginBase):
# @PURPOSE: Lists all files and directories in a specific category and subpath.
# @PARAM: category (Optional[FileCategory]) - The category to list.
# @PARAM: subpath (Optional[str]) - Nested path within the category.
# @PARAM: recursive (bool) - Whether to scan nested subdirectories recursively.
# @PRE: Storage root must exist.
# @POST: Returns a list of StoredFile objects.
# @RETURN: List[StoredFile] - List of file and directory metadata objects.
def list_files(self, category: Optional[FileCategory] = None, subpath: Optional[str] = None) -> List[StoredFile]:
def list_files(
self,
category: Optional[FileCategory] = None,
subpath: Optional[str] = None,
recursive: bool = False,
) -> List[StoredFile]:
with belief_scope("StoragePlugin:list_files"):
root = self.get_storage_root()
logger.info(f"[StoragePlugin][Action] Listing files in root: {root}, category: {category}, subpath: {subpath}")
logger.info(
f"[StoragePlugin][Action] Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}"
)
files = []
categories = [category] if category else list(FileCategory)
@@ -235,17 +243,37 @@ class StoragePlugin(PluginBase):
continue
logger.debug(f"[StoragePlugin][Action] Scanning directory: {target_dir}")
if recursive:
for current_root, dirs, filenames in os.walk(target_dir):
dirs[:] = [d for d in dirs if "Logs" not in d]
for filename in filenames:
file_path = Path(current_root) / filename
if "Logs" in str(file_path):
continue
stat = file_path.stat()
files.append(
StoredFile(
name=filename,
path=str(file_path.relative_to(root)),
size=stat.st_size,
created_at=datetime.fromtimestamp(stat.st_ctime),
category=cat,
mime_type=None,
)
)
continue
# Use os.scandir for better performance and to distinguish files vs dirs
with os.scandir(target_dir) as it:
for entry in it:
# Skip logs
if "Logs" in entry.path:
continue
stat = entry.stat()
is_dir = entry.is_dir()
files.append(StoredFile(
name=entry.name,
path=str(Path(entry.path).relative_to(root)),
@@ -341,4 +369,4 @@ class StoragePlugin(PluginBase):
# [/DEF:get_file_path:Function]
# [/DEF:StoragePlugin:Class]
# [/DEF:StoragePlugin:Module]
# [/DEF:StoragePlugin:Module]