Fixed some problems with this

This commit is contained in:
2026-07-28 13:44:42 +01:00
parent ebebd5d4a5
commit 290f074d74
7 changed files with 145 additions and 110 deletions
+15 -6
View File
@@ -2,16 +2,27 @@ playbook_name: "test_playbook"
registries: registries:
- "./registry/directory_actions.py" - "./registry/directory_actions.py"
- "./registry/restic_actions.py" - "./registry/restic_actions.py"
- "./registry/db_pg_actions.py" - "./registry/dbpg_actions.py"
- "./registry/docker_actions.py"
log_dir: "./" log_dir: "./"
global_context:
repoPath: "/home/esilva/Trash/test_borgmatic/destiny2"
passwordFile: "/home/esilva/Trash/test_borgmatic/pass_file"
steps: steps:
- name: "dump rusty-packages db" - name: "dump rusty-packages db"
actions: actions:
get_container_name: "get_service_container" dump_database: "dump_pg_database"
# dump_database: "dump_pg_database"
context: context:
serviceName: "something" serviceName: "gitea-db"
tmpPathPrefix: "/tmp/db_dump.sql"
- name: "backup db to restic repo"
actions:
dump_database: "backup_data_to_restic_repo"
context:
sourcePath: "/tmp/db_dump.sql"
serviceTag: "gitea-db"
# - name: "backup rusty-packages" # - name: "backup rusty-packages"
# actions: # actions:
# create_repo_dir_if_not_exists: "create_dir_if_not_exists" # create_repo_dir_if_not_exists: "create_dir_if_not_exists"
@@ -19,7 +30,5 @@ steps:
# check_restic_environment: "check_restic_environment" # check_restic_environment: "check_restic_environment"
# backup_data_to_restic_repo: "backup_data_to_restic_repo" # backup_data_to_restic_repo: "backup_data_to_restic_repo"
# context: # context:
# passwordFile: "/home/esilva/Trash/test_borgmatic/pass_file"
# repoPath: "/home/esilva/Trash/test_borgmatic/destiny2"
# sourcePath: "/home/esilva/Trash/rusty_packages" # sourcePath: "/home/esilva/Trash/rusty_packages"
# serviceTag: "rusty-packages" # serviceTag: "rusty-packages"
+80 -45
View File
@@ -6,7 +6,7 @@ import yaml
import json import json
import importlib.util import importlib.util
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Dict, List, Any, Optional from typing import Dict, List, Any, Optional, Tuple
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
@@ -17,7 +17,7 @@ from playbook.premade_steps_registry import registry
class StepIF(ABC): class StepIF(ABC):
@abstractmethod @abstractmethod
def run(self) -> StepLog: def run(self, ctx: Dict[str, Any]) -> StepLog:
"""Perform the actions.""" """Perform the actions."""
pass pass
@@ -39,12 +39,12 @@ class CustomStep(StepIF):
return [action.__name__ for action in self.__actions] return [action.__name__ for action in self.__actions]
# NOTE: refactor this method to be more readable # NOTE: refactor this method to be more readable
def run(self) -> StepLog: def run(self, ctx) -> StepLog:
""" Run the step. """ """ Run the step. """
substeps: List[StepLog] = [] substeps: List[StepLog] = []
status: Status = Status.GOOD status: Status = Status.GOOD
for op in self.__actions: for op in self.__actions:
log: StepLog = timed_run(op, self.__context, op.__name__) log: StepLog = timed_run(op, ctx | self.__context, op.__name__)
status = Status.BAD if log.failed else Status.GOOD status = Status.BAD if log.failed else Status.GOOD
substeps.append(log) substeps.append(log)
@@ -70,6 +70,20 @@ class PlaybookError(Exception):
pass pass
class ContextWrapper:
"""Safely extract values with error messages."""
def __init__(self, ctx: Dict[str, Any]):
self._ctx = ctx
def get(self, key: str, default: Any = None) -> Any:
return self._ctx.get(key, default)
def require(self, *keys: str) -> Tuple[List[str], Dict[str, Any]]:
missing = [k for k in keys if not self._ctx.get(k)]
values = {k: self._ctx.get(k) for k in keys if k not in missing}
return missing, values
@dataclass @dataclass
class StepEntry(object): class StepEntry(object):
name: str name: str
@@ -77,14 +91,16 @@ class StepEntry(object):
class Play(object): class Play(object):
def __init__(self, name : str, registries: Optional[List[ActionRegistry]] = None): def __init__(self, name : str, registries: Dict[str, ActionRegistry]):
self.name: str = name self.name: str = name
self.log_dir: str = "" self.log_dir: str = ""
self.__steps: List[StepEntry] = [] self._context: Dict[str, Any] = {}
self.__registries: List[ActionRegistry] = [registry] + (registries or []) self._steps: List[StepEntry] = []
self._registries: Dict[str, ActionRegistry] = {registry.name: registry} | registries
# NOTE: this should move to the ActionRegistry thing probably
@staticmethod @staticmethod
def __load_registry_file(file_path: str) -> List[ActionRegistry]: def _load_registry_file(file_path: str) -> List[ActionRegistry]:
abs_path = os.path.abspath(file_path) abs_path = os.path.abspath(file_path)
module_name = os.path.splitext(os.path.basename(abs_path))[0] module_name = os.path.splitext(os.path.basename(abs_path))[0]
@@ -107,18 +123,18 @@ class Play(object):
return discovered_registries return discovered_registries
def add_step(self, name: str, stepRunner : StepIF): def add_step(self, name: str, stepRunner : StepIF):
self.__steps.append(StepEntry(name=name, step=stepRunner)) self._steps.append(StepEntry(name=name, step=stepRunner))
def view_playbook(self) -> str: def view_playbook(self) -> str:
steps_info = [] steps_info = []
for s in self.__steps: for s in self._steps:
steps_info.append({ steps_info.append({
"name": s.name, "name": s.name,
"actions": s.step.get_action_names() "actions": s.step.get_action_names()
}) })
registry_data = [] registry_data = []
for reg in self.__registries: for reg in self._registries.values():
registry_data.append(reg.manifest()) registry_data.append(reg.manifest())
data = { data = {
@@ -131,13 +147,13 @@ class Play(object):
return json.dumps(data) return json.dumps(data)
def play(self) -> StepLog: def play(self) -> StepLog:
log: StepLog = timed_run(self.__play) log: StepLog = timed_run(self._play)
if self.log_dir != "": if self.log_dir != "":
log.log_file_path = os.path.join( log.log_file_path = os.path.join(
self.log_dir, self.log_file_name(log.start_date_timestamp) self.log_dir, self.log_file_name(log.start_date_timestamp)
) )
self.__write_log(log) self._write_log(log)
return log return log
@@ -150,16 +166,15 @@ class Play(object):
return dt.strftime(f"{self.name}_%Y-%m-%d_%H-%M-%S.{ms:03d}.log") return dt.strftime(f"{self.name}_%Y-%m-%d_%H-%M-%S.{ms:03d}.log")
def __write_log(self, log: StepLog): def _write_log(self, log: StepLog):
print(log, file=sys.stderr)
try: try:
with open(log.log_file_path, "w") as file: with open(log.log_file_path, "w") as file:
json.dump(asdict(log), file) json.dump(asdict(log), file)
except Exception as e: except Exception as e:
return f"failed to create log file {e}" raise PlaybookError(f"failed to create log file {e}") from e
return "" return ""
def __play(self) -> StepLog: def _play(self) -> StepLog:
playLog: StepLog = StepLog( playLog: StepLog = StepLog(
step_name=self.name, step_name=self.name,
status=Status.GOOD, status=Status.GOOD,
@@ -169,8 +184,8 @@ class Play(object):
substeps=[], substeps=[],
) )
for s in self.__steps: for s in self._steps:
log: StepLog = timed_run(s.step.run) log: StepLog = timed_run(s.step.run, self._context)
playLog.substeps.append(log) playLog.substeps.append(log)
@@ -184,8 +199,8 @@ class Play(object):
return playLog return playLog
def __get_function_from_registry(self, fn_name) -> ActionFn: def _get_function_from_registry(self, fn_name) -> ActionFn:
for reg in self.__registries: for reg in self._registries.values():
try: try:
return reg.get(fn_name, "*") return reg.get(fn_name, "*")
except ValueError: except ValueError:
@@ -193,17 +208,18 @@ class Play(object):
raise ValueError(f"no function with name '{fn_name}' was found") raise ValueError(f"no function with name '{fn_name}' was found")
def __build_custom_step(self, name, actions, ctx) -> CustomStep: def _build_custom_step(self, name, actions, ctx) -> CustomStep:
functionList: List[ActionFn] = [] functionList: List[ActionFn] = []
for f in actions: for f in actions:
try: try:
fn = self.__get_function_from_registry(actions[f]) fn = self._get_function_from_registry(actions[f])
functionList.append(fn) functionList.append(fn)
except KeyError as e: except KeyError as e:
raise PlaybookError(f"Step '{name}' is missing required action field: {e}") from e raise PlaybookError(f"Step '{name}' is missing required action field: {e}") from e
return CustomStep(name, functionList, ctx) return CustomStep(name, functionList, self._context | ctx)
# NOTE: Move to factory
@classmethod @classmethod
def from_yaml(cls, fp: str) -> Play: def from_yaml(cls, fp: str) -> Play:
""" Loads the playbook from a given YAML file. """ """ Loads the playbook from a given YAML file. """
@@ -218,33 +234,26 @@ class Play(object):
yaml_dir = os.path.dirname(os.path.abspath(fp)) yaml_dir = os.path.dirname(os.path.abspath(fp))
return cls.__from_yaml_str(file_contents, base_dir=yaml_dir) return cls.__from_yaml_str(file_contents, base_dir=yaml_dir)
# NOTE: Move to factory
@classmethod @classmethod
def __from_yaml_str(cls, yaml_str: str, base_dir: str = ".") -> Play: def __from_yaml_str_load_registries(cls, dir, data) -> Dict[str, ActionRegistry]:
data = None custom_registries: Dict[str, ActionRegistry] = {}
try: registry_paths = data.get("registries", {})
data = yaml.safe_load(yaml_str)
except Exception as e:
raise PlaybookError(f"YAML Syntax error: {e}") from e
if not isinstance(data, dict) or "playbook_name" not in data:
raise PlaybookError("YAML must contain a top-level 'playbook_name' field.")
# LOAD THE CUSTOM REGISTRIES
custom_registries: List[ActionRegistry] = []
registry_paths = data.get("registries", [])
custom_registries: Dict[str, ActionRegistry] = {}
for reg_path in registry_paths: for reg_path in registry_paths:
full_path = os.path.join(base_dir, reg_path) if not os.path.isabs(reg_path) else reg_path full_path = os.path.join(dir, reg_path) if not os.path.isabs(reg_path) else reg_path
try: try:
regs = cls.__load_registry_file(full_path) regs = cls._load_registry_file(full_path)
custom_registries.extend(regs) for new_reg in regs:
custom_registries[new_reg.name] = new_reg
except Exception as e: except Exception as e:
raise PlaybookError(f"Error loading registry '{reg_path}': {e}") from e raise PlaybookError(f"Error loading registry '{reg_path}': {e}") from e
playbook = cls(data["playbook_name"], registries=custom_registries) return custom_registries
if data["log_dir"]:
playbook.log_dir = data["log_dir"]
# NOTE: Move to factory
def __from_yaml_str_load_steps(self, data):
for idx, step_cfg in enumerate(data.get("steps", [])): for idx, step_cfg in enumerate(data.get("steps", [])):
step_name = step_cfg.get("name", f"step_{idx}") step_name = step_cfg.get("name", f"step_{idx}")
actions = step_cfg.get("actions", {}) actions = step_cfg.get("actions", {})
@@ -256,9 +265,35 @@ class Play(object):
) )
try: try:
customStep: CustomStep = playbook.__build_custom_step(step_name, actions, context) customStep: CustomStep = self._build_custom_step(step_name, actions, context)
playbook.add_step(step_cfg["name"], customStep) self.add_step(step_cfg["name"], customStep)
except Exception as e: except Exception as e:
raise PlaybookError(f"Error configuring step '{step_name}': {e}") from e raise PlaybookError(f"Error configuring step '{step_name}': {e}") from e
# NOTE: Move to factory
@classmethod
def __from_yaml_str(cls, yaml_str: str, base_dir: str = ".") -> Play:
data = None
try:
data = yaml.safe_load(yaml_str)
except Exception as e:
raise PlaybookError(f"YAML Syntax error: {e}") from e
if not isinstance(data, dict) or "playbook_name" not in data:
raise PlaybookError("YAML must contain a top-level 'playbook_name' field.")
try:
custom_registries = cls.__from_yaml_str_load_registries(base_dir, data)
except Exception as e:
raise PlaybookError(f"registries load error: {e}") from e
playbook = cls(data["playbook_name"], registries=custom_registries)
try:
playbook.__from_yaml_str_load_steps(data)
except Exception as e:
raise PlaybookError(f"steps load error: {e}") from e
playbook._context = data.get("global_context", {})
playbook.log_dir = data.get("log_dir", "")
return playbook return playbook
+1
View File
@@ -25,6 +25,7 @@ class StepLog(object):
error: List[Dict[Any, Any]] = field(default_factory=list) error: List[Dict[Any, Any]] = field(default_factory=list)
substeps: List[StepLog] = field(default_factory=list) substeps: List[StepLog] = field(default_factory=list)
log_file_path: str = "" log_file_path: str = ""
pipe_ctx: Dict[str, Any] = field(default_factory=dict)
@property @property
def failed(self) -> bool: def failed(self) -> bool:
-32
View File
@@ -1,32 +0,0 @@
# restic actions registry
from playbook.action_registry import ActionRegistry
from playbook.models import StepLog
import sys
import json
import docker
import subprocess
from typing import List, Dict, Any, Tuple
dbpgactions = ActionRegistry("dbpg_actions_reg")
@dbpgactions.register(name="dump_pg_database", version="")
def dump_pg_database(ctx, name) -> StepLog:
return StepLog.ok(name, "")
# NOTE: this functions ought to be in their own registry, but I need inter
# registry communication to be possible first
###############################################################################
##### DOCKER FUNCTIONS ########################################################
@dbpgactions.register(name="get_service_container", version="")
def get_service_container(ctx, name) -> StepLog:
client = docker.from_env()
filters: Dict[str, Any]= {"label": [f"com.docker.compose.service=gitea-db"]}
print(client.containers.list(filters=filters, all=True), file=sys.stderr)
return StepLog.ok(name, "cool beans")
+21
View File
@@ -0,0 +1,21 @@
# restic actions registry
from playbook.action_registry import ActionRegistry
from playbook.models import StepLog
from playbook import ContextWrapper
from registry.docker_actions import dockeractions
import sys
import json
import docker
import subprocess
from typing import List, Dict, Any, Tuple
dbpgactions = ActionRegistry("dbpg_actions_reg")
@dbpgactions.register(name="dump_pg_database", version="")
def dump_pg_database(ctx, name) -> StepLog:
c = ContextWrapper(ctx)
return StepLog.ok(name, "")
+21
View File
@@ -0,0 +1,21 @@
from playbook.action_registry import ActionRegistry
from playbook.models import StepLog
import sys
import json
import docker
import subprocess
from typing import List, Dict, Any, Tuple
dockeractions = ActionRegistry("docker_actions_reg")
@dockeractions.register(name="get_service_container", version="")
def get_service_container(ctx, name) -> StepLog:
client = docker.from_env()
filters: Dict[str, Any]= {"label": [f"com.docker.compose.service=gitea-db"]}
print(client.containers.list(filters=filters, all=True), file=sys.stderr)
return StepLog.ok(name, "cool beans")
+7 -27
View File
@@ -114,7 +114,7 @@ def create_repo_if_not_exists(ctx, name: str = "") -> StepLog:
return StepLog.fail(name, [{"status": "failed", "output": log.error}]) return StepLog.fail(name, [{"status": "failed", "output": log.error}])
# Create repo # Create repo
repoCreateLog: StepLog = create_restic_repo(ctx, name + f".{create_restic_repo}") repoCreateLog: StepLog = create_restic_repo(ctx, name + f".{create_restic_repo.__name__}")
if repoCreateLog.failed: if repoCreateLog.failed:
return StepLog.fail(name, [{"status": "failed", "output": repoCreateLog.error}]) return StepLog.fail(name, [{"status": "failed", "output": repoCreateLog.error}])
msg = "repo created" msg = "repo created"
@@ -146,13 +146,10 @@ def check_if_repo_exists(ctx, name: str = "") -> StepLog:
@restic.register(name="backup_data_to_restic_repo", version="") @restic.register(name="backup_data_to_restic_repo", version="")
def backup_data_to_restic_repo(ctx, name) -> StepLog: def backup_data_to_restic_repo(ctx, name) -> StepLog:
out = subprocess.run( cmd = ["restic", "backup", ctx["sourcePath"], "--json", "--quiet",
["restic", "backup", ctx["sourcePath"], "--json", "--quiet", "-r", ctx["repoPath"], "--password-file", ctx["passwordFile"]]
"-r", ctx["repoPath"], "--password-file", ctx["passwordFile"]],
text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
)
json_output: List[Dict[str, Any]] = parse_output(out.stdout) returncode, json_output = run_restic_command(cmd)
errors: List[Dict[str, Any]] = find_restic_errors(json_output) errors: List[Dict[str, Any]] = find_restic_errors(json_output)
# Handle error messages in JSON output # Handle error messages in JSON output
@@ -160,24 +157,7 @@ def backup_data_to_restic_repo(ctx, name) -> StepLog:
return StepLog.fail(name, errors) return StepLog.fail(name, errors)
# Make sure the return code is good too # Make sure the return code is good too
if out.returncode != 0: if returncode != 0:
return StepLog.fail(name, [{"status": "failed", "output": out.stdout}]) return StepLog.fail(name, [{"status": "failed", "output": errors}])
# Return code is 0, so it was successful, output the message return StepLog.ok(name, "backup successful")
try:
data = json.loads(out.stdout)
msg = data
except:
msg = {
"status": "ran succesfully, but an error occurred parsing the output",
"output": out.stdout,
}
return StepLog.ok(name, str(msg))
#-- BIG NOTE ------------------------------------------------------------------#
# I will use the restic registry for database actions until I make it so cross #
# registry calls with context are possible. But right now the yaml config #
# doesnt support that anyway. #
#------------------------------------------------------------------------------#