diff --git a/playbook.yaml b/playbook.yaml index 41714ad..7450c30 100644 --- a/playbook.yaml +++ b/playbook.yaml @@ -1,4 +1,4 @@ -playbook_name: "test_playbook" +playbook_name: "gitea_service_backup" registries: - "./registry/directory_actions.py" - "./registry/restic_actions.py" @@ -11,30 +11,37 @@ global_context: passwordFile: "/home/esilva/Trash/test_borgmatic/pass_file" steps: - - name: "dump rusty-packages db" + - name: "find database container" actions: - dump_database: "dump_pg_database" + find_container: "get_service_container_name" context: - serviceName: "gitea-db" - tmpPathPrefix: "/tmp/db_dump.sql" - - name: "test context" + service_name: "gitea-db" + + - name: "dump database" actions: - dump_database: "dump_pg_database" + dump_database: "dump_container_pg_db" context: - serviceName: "gitea-db" - tmpPathPrefix: "/tmp/db_dump.sql" - # - name: "backup db to restic repo" - # actions: - # dump_database: "backup_data_to_restic_repo" - # context: - # sourcePath: "/tmp/db_dump.sql" - # serviceTag: "gitea-db" - # - name: "backup rusty-packages" - # actions: - # create_repo_dir_if_not_exists: "create_dir_if_not_exists" - # create_repo_if_not_exists: "create_repo_if_not_exists" - # check_restic_environment: "check_restic_environment" - # backup_data_to_restic_repo: "backup_data_to_restic_repo" - # context: - # sourcePath: "/home/esilva/Trash/rusty_packages" - # serviceTag: "rusty-packages" + dump_path: "/tmp/db_dump.sql" + db_user: "gitea" + database: "gitea" + + - name: "restic database backup" + actions: + dump_database: "backup_data_to_restic_repo" + context: + source_path: "/tmp/db_dump.sql" + tags: ["gitea", "db-dump"] + + - name: "backup gitea mount points data" + actions: + backup_data_to_restic_repo: "backup_data_to_restic_repo" + context: + source_path: "/home/esilva/Trash/docker-compose/resources/gitea/gitea_data/" + tags: ["gitea", "data"] + + - name: "backup gitea database mount points data" + actions: + backup_data_to_restic_repo: "backup_data_to_restic_repo" + context: + source_path: "/home/esilva/Trash/docker-compose/resources/gitea/postgres_data/" + tags: ["gitea", "db-data"] diff --git a/registry/dbpg_actions.py b/registry/dbpg_actions.py index c666259..564ae23 100644 --- a/registry/dbpg_actions.py +++ b/registry/dbpg_actions.py @@ -16,7 +16,7 @@ dbpgactions = ActionRegistry("dbpg_actions_reg") @dbpgactions.register(name="dump_pg_database", version="") -@ContextChecker.requires("database") -def dump_pg_database(ctx, name) -> StepLog: +@ContextChecker.requires("database", "compose") +def dump_pg_database_from_container(ctx, name) -> StepLog: print(ctx, file=sys.stderr) return StepLog.ok(name, "", pipe_ctx={"new_data": "coolData"}) diff --git a/registry/docker_actions.py b/registry/docker_actions.py index afd4f91..7f91c82 100644 --- a/registry/docker_actions.py +++ b/registry/docker_actions.py @@ -1,21 +1,65 @@ +from os import error + +from yaml import dump + +from playbook import models from playbook.action_registry import ActionRegistry -from playbook.models import StepLog +from playbook.models import StepLog, ContextChecker import sys import json import docker import subprocess from typing import List, Dict, Any, Tuple +# from docker.models.containers import Container dockeractions = ActionRegistry("docker_actions_reg") -@dockeractions.register(name="get_service_container", version="") -def get_service_container(ctx, name) -> StepLog: +@dockeractions.register(name="get_service_container_name", version="") +@ContextChecker.requires("service_name") +def get_service_container_name(ctx, name) -> StepLog: client = docker.from_env() - filters: Dict[str, Any]= {"label": [f"com.docker.compose.service=gitea-db"]} - print(client.containers.list(filters=filters, all=True), file=sys.stderr) + filters: Dict[str, Any]= { + "label": [f"com.docker.compose.service={ctx["service_name"]}"] + } + containers = client.containers.list(filters=filters, all=True) - return StepLog.ok(name, "cool beans") + if not containers: + return StepLog.fail(name, [{ + "status": "failed", + "output": f"no container found for service {ctx["service_name"]}"} + ]) + + new_data = {"container": containers[0].id} + return StepLog.ok(name, f"found container {containers[0].id}", pipe_ctx=new_data) + + +@dockeractions.register(name="dump_container_pg_db", version="") +@ContextChecker.requires("container", "dump_path", "db_user", "database") +def dump_container_pg_db(ctx, name) -> StepLog: + client = docker.from_env() + container = client.containers.get(ctx["container"]) + + res = container.exec_run( + cmd=f"pg_dump -U {ctx["db_user"]} {ctx["database"]}", + stream=False, # change to True to stream directly to host file (need to check) + demux=True, + ) + stdout, stderr = res.output + + if stderr: + if not isinstance(stderr, bytes): + return StepLog.fail(name, [{"status": "failed", "output": "stderr is not bytes"}]) + if res.exit_code != 0: + error_msg = stderr.decode("utf-8") if stderr else "dump failed with no output" + return StepLog.fail(name, [{"status": "failed", "output": error_msg}]) + + if not isinstance(stdout, bytes): + return StepLog.fail(name, [{"status": "failed", "output": "stdout is not bytes"}]) + with open(ctx["dump_path"], "wb") as f: + f.write(stdout) + + return StepLog.ok(name, "database dump successful") diff --git a/registry/restic_actions.py b/registry/restic_actions.py index 5784c98..f625729 100644 --- a/registry/restic_actions.py +++ b/registry/restic_actions.py @@ -1,6 +1,6 @@ # restic actions registry from playbook.action_registry import ActionRegistry -from playbook.models import StepLog +from playbook.models import ContextChecker, StepLog import sys import json @@ -145,11 +145,15 @@ def check_if_repo_exists(ctx, name: str = "") -> StepLog: @restic.register(name="backup_data_to_restic_repo", version="") +@ContextChecker.requires("source_path", "passwordFile") def backup_data_to_restic_repo(ctx, name) -> StepLog: - cmd = ["restic", "backup", ctx["sourcePath"], "--json", "--quiet", + cmd = ["restic", "backup", ctx["source_path"], "--json", "--quiet", "-r", ctx["repoPath"], "--password-file", ctx["passwordFile"]] + if ctx["tags"]: + cmd.extend(["--tag", ','.join(ctx["tags"])]) returncode, json_output = run_restic_command(cmd) + print(json_output, file=sys.stderr) errors: List[Dict[str, Any]] = find_restic_errors(json_output) # Handle error messages in JSON output diff --git a/taskrunner.py b/taskrunner.py index 5957739..8187ee6 100644 --- a/taskrunner.py +++ b/taskrunner.py @@ -33,7 +33,7 @@ def main(): traceback.print_exc() exit(1) - print(newPlay.view_playbook()) + # print(newPlay.view_playbook()) print(json.dumps(asdict(newPlay.play()), indent=2))