From 79a3dab7348065b0f577937a36c451e43d64825a Mon Sep 17 00:00:00 2001 From: Paul Wilde Date: Mon, 19 Aug 2024 02:21:46 +0100 Subject: [PATCH] fix misfile --- norg.nimble | 2 +- norg.toml.sample | 51 +++++++++++++ norg/borg/borg.nim | 111 ++--------------------------- norg/borg/create.nim | 45 ++++++++++++ norg/borg/execute.nim | 28 ++++++++ norg/borg/extract.nim | 24 +++++++ norg/borg/mount.nim | 19 +++++ norg/borg/prune.nim | 19 +++++ norg/config/actions_config.nim | 44 ++++++++++++ norg/config/init.nim | 9 ++- norg/config/maintenance_config.nim | 11 +++ norg/model/config_type.nim | 19 +---- norg/model/encryption_type.nim | 18 +++++ norg/model/maintenance_type.nim | 17 +++++ norg/model/uptimekuma_type.nim | 5 +- norg/notifier/notifier.nim | 4 +- norg/utils/actions.nim | 2 +- readme.md | 27 ++++++- todo.md | 4 +- 19 files changed, 327 insertions(+), 132 deletions(-) create mode 100644 norg.toml.sample create mode 100644 norg/borg/create.nim create mode 100644 norg/borg/execute.nim create mode 100644 norg/borg/extract.nim create mode 100644 norg/borg/mount.nim create mode 100644 norg/borg/prune.nim create mode 100644 norg/config/actions_config.nim create mode 100644 norg/config/maintenance_config.nim create mode 100644 norg/model/encryption_type.nim create mode 100644 norg/model/maintenance_type.nim diff --git a/norg.nimble b/norg.nimble index e6012ea..274c064 100644 --- a/norg.nimble +++ b/norg.nimble @@ -1,6 +1,6 @@ # Package -version = "0.1.1" +version = "0.1.5" author = "Paul Wilde" description = "A Borg Backup Wrapper" license = "AGPL-3.0-or-later" diff --git a/norg.toml.sample b/norg.toml.sample new file mode 100644 index 0000000..790af25 --- /dev/null +++ b/norg.toml.sample @@ -0,0 +1,51 @@ +# Source Directories you would like to back up in a TOML list format +source_directories = [ + "/home/me/Music", + "/home/me/Pictures" +] + +# Repositories to back up to, in separate TOML objects +[[repositories]] +label = "A Repository" +path = "/my/backup/location" +[[repositories]] +label = "Another Respository at BorgBase" +path = "ssh://1234abcd@1234abcd.repo.borgbase.com/./repo" + +# Encryption Configuration +[encryption] +# `encryption_passphrase` gets set as the `BORG_PASSPHRASE` env var +encryption_passphrase = "MyReallySecurePassword" +# I hope to add more Borg encryption env vars here in time + +# Actions that are called at various times during runtime +[actions] +# "everything" means before or after every possible option for all repositories +before_everything = ["echo before everything"] +after_everything = ["echo after everything"] +# "actions" means before any action, per repository +before_actions = ["echo before actions"] +after_actions = ["echo after actions"] +# before or after the backup process per repository +before_backup = ["echo before backup"] +after_backup = ["echo after backup"] +# before or after the extract process per repository +before_extract = ["echo before extract"] +after_extract = ["echo after extract"] +# before or after the prune process per repository +before_prune = ["echo before prune"] +after_prune = ["echo after prune"] +# before or after the compact process per respository +before_compact = ["echo before compact"] +after_compact = ["echo after compact"] +# before or after the check processs per repository +before_check = ["echo before check"] +after_check = ["echo after check"] + +[uptimekuma] +# The base/push url of your Uptime Kuma monitor - without the query string. +# The quenry string will be generated at run time and will change dependant on the state of your backup. +base_url = "https://uptime.kuma.url/api/push/1234abcd" + +# what backup states you wish to send an alert for, defaults to Success, Failure and Running +states = ["Success","Failure", "Running"] diff --git a/norg/borg/borg.nim b/norg/borg/borg.nim index 1d42578..4f192f0 100644 --- a/norg/borg/borg.nim +++ b/norg/borg/borg.nim @@ -1,118 +1,21 @@ import ../model/config_type -import ../model/state_type +import ../model/encryption_type import ../model/borg_type -import ../notifier/notifier import ../utils/actions -import strutils -import strformat -import sequtils -import osproc -import times -import os -import nativesockets +import execute +import prune +import create +import mount +import extract -proc genArchiveName(): string = - let hostname = getHostname() - let ts = getTime().format("yyyy-MM-dd'T'HH:mm:ss'.'ffffff") - return fmt"{hostname}-{ts}" - -proc genCommand(cmd: string, repo: string, others: seq[string]): string = - let args = others.join(" ") - let cmd = fmt"{BORG_BIN} {cmd} {repo} {args}" - return cmd - -proc run(cmd: string): int = - echo fmt"Trying to run : {cmd}" - try: - let res = execProcess(cmd) - echo res - except: - echo getCurrentExceptionMsg() - -proc runDiscard(cmd: string): int = - echo fmt"Trying to run : {cmd}" - try: - let res = execCmd(cmd) - return res - except: - echo getCurrentExceptionMsg() - return 1 proc initRepo(nc: NorgConfig, repo: Repository): int = return runDiscard genCommand(cmd = "init", repo = repo.path, others = nc.args.others) -proc createArchive(nc: NorgConfig, repo: Repository, archivename: string, retry: int = 0): int = - let others = concat(nc.source_directories, nc.args.others) - let res = run genCommand(cmd = "create", repo = archivename, others = others) - if res == 1: - sleep 15 * 1000 # 15 seconds - if retry == nc.retries: - return 1 - else: - return createArchive(nc, repo, archivename, retry + 1) - return res - -proc backupSources(nc: NorgConfig, repo: Repository): int = - let start_time = now() - notify(nc.notifiers, state=Running) - let end_time = now() - let archivename = repo.path & "::" & genArchiveName() - let res = createArchive(nc, repo, archivename) - let total = (end_time - start_time).inMilliSeconds() - case res - of 0: - notify(nc.notifiers, state=Success, runtime=total) - of 1: - notify(nc.notifiers, state=Failure, runtime=total) - else: - notify(nc.notifiers, state=Failure, runtime=total) - return res - - proc listArchives(nc: NorgConfig, repo: Repository): int = return run genCommand(cmd = "list", repo = repo.path, others = nc.args.others) -proc mountArchive(nc: NorgConfig, repo: Repository): int = - let archive = repo.path & "::" & nc.args.others[0] - let others = nc.args.others[1..^1] - let ok = runDiscard genCommand(cmd = "mount", repo = archive, others = others) - if ok == 0: - echo fmt"Mounted {archive} at {others[0]}" - else: - echo "Failed to mount ", archive - -proc unmountArchive(nc: NorgConfig): int = - let ok = runDiscard genCommand(cmd = "umount", repo = "", others = nc.args.others) - if ok == 0: - echo "Unmounted ", nc.args.others[0] - else: - echo "Failed to unmount ", nc.args.others[0] - -proc isEmpty(dir: string): bool = - var count = 0 - for idx, f in walkDir(dir): return false - #count += 1 - return count == 0 - -proc extractArchive(nc: NorgConfig, repo: Repository): int = - let archive = fmt"{repo.path}::{nc.args.others[0]}" - var others = nc.args.others[1..^1] - if nc.args.extract_destination != "": - discard existsOrCreateDir(nc.args.extract_destination) - setCurrentDir(nc.args.extract_destination) - let dir = getCurrentDir() - if dir.isEmpty(): - echo "Restoring..." - let ok = run(genCommand(cmd = "extract", repo = archive, others = others)) - return ok - else: - echo "Not restoring to non-empty destination\r\nPlease use the --destination flag" - -proc pruneRepo(nc: NorgConfig, repo: Repository): int = - echo "Not Yet Implemented." - discard - proc compactRepo(nc: NorgConfig, repo: Repository): int = echo "Not Yet Implemented." discard @@ -130,7 +33,7 @@ proc execute*(nc: NorgConfig) = discard initRepo(nc, repo) of CREATE: run_actions(norg_config.actions.before_backup) - discard backupSources(nc, repo) + discard createBackup(nc, repo) run_actions(norg_config.actions.after_backup) of LIST: discard listArchives(nc, repo) diff --git a/norg/borg/create.nim b/norg/borg/create.nim new file mode 100644 index 0000000..9f353a6 --- /dev/null +++ b/norg/borg/create.nim @@ -0,0 +1,45 @@ +import ../model/config_type +import ../model/state_type +import ../notifier/notifier + +import execute +import prune + +import os +import times +import sequtils +import strformat +import nativesockets + +proc genArchiveName(): string = + let hostname = getHostname() + let ts = getTime().format("yyyy-MM-dd'T'HH:mm:ss'.'ffffff") + return fmt"{hostname}-{ts}" + +proc createArchive(nc: NorgConfig, repo: Repository, archivename: string, retry: int = 0): int = + let others = concat(nc.source_directories, nc.args.others) + let res = run genCommand(cmd = "create", repo = archivename, others = others) + if res != 0: + sleep 15 * 1000 # 15 seconds + if retry == nc.retries: + return 1 + else: + return createArchive(nc, repo, archivename, retry + 1) + return res + +proc createBackup*(nc: NorgConfig, repo: Repository): int = + let start_time = now() + notify(nc.notifiers, state=Running) + let end_time = now() + let archivename = repo.path & "::" & genArchiveName() + let res = createArchive(nc, repo, archivename) + let total = (end_time - start_time).inMilliSeconds() + case res + of 0: + discard pruneRepo(nc, repo) + notify(nc.notifiers, state=Success, runtime=total) + of 1: + notify(nc.notifiers, state=Failure, runtime=total) + else: + notify(nc.notifiers, state=Failure, runtime=total, msg = $res) + return res diff --git a/norg/borg/execute.nim b/norg/borg/execute.nim new file mode 100644 index 0000000..099ac7a --- /dev/null +++ b/norg/borg/execute.nim @@ -0,0 +1,28 @@ +import strutils +import strformat +import osproc + +import ../model/borg_type + +proc genCommand*(cmd: string, repo: string, others: seq[string]): string = + let args = others.join(" ") + let cmd = fmt"{BORG_BIN} {cmd} {repo} {args}" + return cmd + +proc run*(cmd: string): int = + echo fmt"Trying to run : {cmd}" + try: + let res = execCmd(cmd) + return res + except: + echo getCurrentExceptionMsg() + return 1 + +proc runDiscard*(cmd: string): int = + echo fmt"Trying to run : {cmd}" + try: + let res = execCmd(cmd) + return res + except: + echo getCurrentExceptionMsg() + return 1 diff --git a/norg/borg/extract.nim b/norg/borg/extract.nim new file mode 100644 index 0000000..e94c930 --- /dev/null +++ b/norg/borg/extract.nim @@ -0,0 +1,24 @@ +import ../model/config_type +import execute +import strformat +import os + +proc isEmpty(dir: string): bool = + var count = 0 + for idx, f in walkDir(dir): return false + #count += 1 + return count == 0 + +proc extractArchive*(nc: NorgConfig, repo: Repository): int = + let archive = fmt"{repo.path}::{nc.args.others[0]}" + var others = nc.args.others[1..^1] + if nc.args.extract_destination != "": + discard existsOrCreateDir(nc.args.extract_destination) + setCurrentDir(nc.args.extract_destination) + let dir = getCurrentDir() + if dir.isEmpty(): + echo "Restoring..." + let ok = run genCommand(cmd = "extract", repo = archive, others = others) + return ok + else: + echo "Not restoring to non-empty destination\r\nPlease use the --destination flag" diff --git a/norg/borg/mount.nim b/norg/borg/mount.nim new file mode 100644 index 0000000..0e9a94a --- /dev/null +++ b/norg/borg/mount.nim @@ -0,0 +1,19 @@ +import ../model/config_type +import execute +import strformat + +proc mountArchive*(nc: NorgConfig, repo: Repository): int = + let archive = repo.path & "::" & nc.args.others[0] + let others = nc.args.others[1..^1] + let ok = runDiscard genCommand(cmd = "mount", repo = archive, others = others) + if ok == 0: + echo fmt"Mounted {archive} at {others[0]}" + else: + echo "Failed to mount ", archive + +proc unmountArchive*(nc: NorgConfig): int = + let ok = runDiscard genCommand(cmd = "umount", repo = "", others = nc.args.others) + if ok == 0: + echo "Unmounted ", nc.args.others[0] + else: + echo "Failed to unmount ", nc.args.others[0] diff --git a/norg/borg/prune.nim b/norg/borg/prune.nim new file mode 100644 index 0000000..688fb1b --- /dev/null +++ b/norg/borg/prune.nim @@ -0,0 +1,19 @@ +import ../model/config_type + +import strformat + +import execute + +proc addPruneOptions(cmd: var string, maintenance: Maintenance) = + cmd = fmt"""{cmd} \ + --keep-hourly {maintenance.keep_hourly} \ + --keep-daily {maintenance.keep_daily} \ + --keep-weekly {maintenance.keep_weekly} \ + --keep-monthly {maintenance.keep_monthly} \ + --keep-yearly {maintenance.keep_yearly} \ + """ + +proc pruneRepo*(nc: NorgConfig, repo: Repository): int = + var cmd = genCommand(cmd = "prune", repo = repo.path, others = nc.args.others) + cmd.addPruneOptions(nc.maintenance) + return run cmd diff --git a/norg/config/actions_config.nim b/norg/config/actions_config.nim new file mode 100644 index 0000000..4ac1981 --- /dev/null +++ b/norg/config/actions_config.nim @@ -0,0 +1,44 @@ +import ../model/actions_type +import parsetoml + +proc parseActions*(conf: TomlValueRef): Actions = + var actions: Actions = Actions() + # Oh I hate this bit.. + + # Everything + for action in conf{"before_everything"}.getElems(): + actions.before_everything.add(action.getStr()) + for action in conf{"after_everything"}.getElems(): + actions.after_everything.add(action.getStr()) + # Actions + for action in conf{"before_actions"}.getElems(): + actions.before_actions.add(action.getStr()) + for action in conf{"after_actions"}.getElems(): + actions.after_actions.add(action.getStr()) + # Backup + for action in conf{"before_backup"}.getElems(): + actions.before_backup.add(action.getStr()) + for action in conf{"after_backup"}.getElems(): + actions.after_backup.add(action.getStr()) + # Extract + for action in conf{"before_extract"}.getElems(): + actions.before_extract.add(action.getStr()) + for action in conf{"after_extract"}.getElems(): + actions.after_extract.add(action.getStr()) + # Prune + for action in conf{"before_prune"}.getElems(): + actions.before_prune.add(action.getStr()) + for action in conf{"after_prune"}.getElems(): + actions.after_prune.add(action.getStr()) + # Compact + for action in conf{"before_compact"}.getElems(): + actions.before_compact.add(action.getStr()) + for action in conf{"after_compact"}.getElems(): + actions.after_compact.add(action.getStr()) + # Check + for action in conf{"before_check"}.getElems(): + actions.before_check.add(action.getStr()) + for action in conf{"after_check"}.getElems(): + actions.after_check.add(action.getStr()) + + return actions diff --git a/norg/config/init.nim b/norg/config/init.nim index 77b2a8c..b310b7d 100644 --- a/norg/config/init.nim +++ b/norg/config/init.nim @@ -1,7 +1,10 @@ import parsetoml import ../model/config_type +import ../model/encryption_type import notifier_config +import actions_config +import maintenance_config export config_type proc parseSourceDirectories*(in_conf: TomlValueRef): seq[string] = @@ -14,7 +17,9 @@ proc parseSourceDirectories*(in_conf: TomlValueRef): seq[string] = return src_dirs proc parseEncryption*(enc_conf: TomlValueRef) = - norg_config.setEncryptionPassphrase(enc_conf{"encryption_passphrase"}.getStr("")) + setEncryptionPassphrase(enc_conf{"encryption_passphrase"}.getStr("")) + setEncryptionPassphraseFD(enc_conf{"encryption_passphrase_fd"}.getStr("")) + setEncryptionPassCommand(enc_conf{"encryption_passcommand"}.getStr("")) proc parseRepositories*(rep_conf: TomlValueRef): seq[Repository] = var repos: seq[Repository] = @[] @@ -33,6 +38,8 @@ proc parseConfigFile*(file: string): NorgConfig = parseEncryption(in_conf{"encryption"}) norg_config.repositories = parseRepositories(in_conf{"repositories"}) norg_config.notifiers = parseNotifiers(in_conf) + norg_config.actions = parseActions(in_conf{"actions"}) + norg_config.maintenance = parseMaintenance(in_conf{"maintenance"}) return norg_config diff --git a/norg/config/maintenance_config.nim b/norg/config/maintenance_config.nim new file mode 100644 index 0000000..d3a1725 --- /dev/null +++ b/norg/config/maintenance_config.nim @@ -0,0 +1,11 @@ +import ../model/maintenance_type +import parsetoml + +proc parseMaintenance*(conf: TomlValueRef): Maintenance = + var maintenance = newMaintenance() + maintenance.keep_hourly = conf{"keep_hourly"}.getInt(maintenance.keep_hourly) + maintenance.keep_daily = conf{"keep_daily"}.getInt(maintenance.keep_daily) + maintenance.keep_weekly = conf{"keep_weekly"}.getInt(maintenance.keep_weekly) + maintenance.keep_monthly = conf{"keep_monthly"}.getInt(maintenance.keep_monthly) + maintenance.keep_yearly = conf{"keep_yearly"}.getInt(maintenance.keep_yearly) + return maintenance diff --git a/norg/model/config_type.nim b/norg/model/config_type.nim index fef8050..5ae2cd3 100644 --- a/norg/model/config_type.nim +++ b/norg/model/config_type.nim @@ -1,12 +1,13 @@ import repository_type import notifier_types import actions_type +import maintenance_type import ../config/args -import os export repository_type export notifier_types export actions_type +export maintenance_type type @@ -19,20 +20,9 @@ type notifiers*: Notifiers actions*: Actions args*: NorgArgs - Maintenance* = object - keep_daily*: int - keep_weekly*: int - keep_monthly*: int var norg_config*: NorgConfig -proc newMaintenance*(): Maintenance = - var m = Maintenance() - m.keep_daily = 7 - m.keep_weekly = 4 - m.keep_monthly = 6 - return m - proc newNorgConfig*(): NorgConfig = var nc = NorgConfig() nc.maintenance = newMaintenance() @@ -44,11 +34,6 @@ proc newNorgConfig*(): NorgConfig = # msg &= "Notifiers: " & $c.notifiers & "\r\n" # return msg -proc setEncryptionPassphrase*(nc: var NorgConfig, pw: string) = - putEnv("BORG_PASSPHRASE", pw) - -proc delEncryptionPassphraseInfo*() = - delEnv("BORG_PASSPHRASE") diff --git a/norg/model/encryption_type.nim b/norg/model/encryption_type.nim new file mode 100644 index 0000000..09381e6 --- /dev/null +++ b/norg/model/encryption_type.nim @@ -0,0 +1,18 @@ +import os + +proc setEncryptionPassphrase*(pw: string) = + if pw != "": + putEnv("BORG_PASSPHRASE", pw) + +proc setEncryptionPassphraseFD*(pw: string) = + if pw != "": + putEnv("BORG_PASSPHRASE_FD", pw) + +proc setEncryptionPassCommand*(pw: string) = + if pw != "": + putEnv("BORG_PASSCOMMAND", pw) + +proc delEncryptionPassphraseInfo*() = + delEnv("BORG_PASSPHRASE") + delEnv("BORG_PASSPHRASE_FD") + delEnv("BORG_PASSCOMMAND") diff --git a/norg/model/maintenance_type.nim b/norg/model/maintenance_type.nim new file mode 100644 index 0000000..d8197d2 --- /dev/null +++ b/norg/model/maintenance_type.nim @@ -0,0 +1,17 @@ + +type + Maintenance* = object + keep_hourly*: int + keep_daily*: int + keep_weekly*: int + keep_monthly*: int + keep_yearly*: int + +proc newMaintenance*(): Maintenance = + var m = Maintenance() + m.keep_hourly = 24 + m.keep_daily = 7 + m.keep_weekly = 4 + m.keep_monthly = 6 + m.keep_yearly = 1 + return m diff --git a/norg/model/uptimekuma_type.nim b/norg/model/uptimekuma_type.nim index f82c1c8..296d05a 100644 --- a/norg/model/uptimekuma_type.nim +++ b/norg/model/uptimekuma_type.nim @@ -7,14 +7,15 @@ import ../utils/httprequest type UptimeKuma* = object of Notifier -proc send_notify*(uk: UptimeKuma, state: State, runtime: int = 0): int {.discardable.} = +proc send_notify*(uk: UptimeKuma, state: State, runtime: int = 0, msg: string = ""): int {.discardable.} = var status: string case state of Success, Running: status = "up" else: status = "down" - let url = fmt"{uk.base_url}?status={status}&msg={state}&ping={runtime}" + let message = fmt"{status}\r\n{msg}" + let url = fmt"{uk.base_url}?status={status}&msg={message}&ping={runtime}" echo "Sending notification to " & url let res = sendHttpRequest(HttpGet, url) diff --git a/norg/notifier/notifier.nim b/norg/notifier/notifier.nim index 0cac445..15387a5 100644 --- a/norg/notifier/notifier.nim +++ b/norg/notifier/notifier.nim @@ -1,7 +1,7 @@ import ../model/state_type import ../model/notifier_types -proc notify*(notifiers: Notifiers, state: State, runtime: int = 0): int {.discardable.} = +proc notify*(notifiers: Notifiers, state: State, runtime: int = 0, msg: string = ""): int {.discardable.} = if notifiers.uptimekuma.base_url != "" and state in notifiers.uptimekuma.states: - notifiers.uptimekuma.send_notify(state, runtime) + notifiers.uptimekuma.send_notify(state, runtime, msg) diff --git a/norg/utils/actions.nim b/norg/utils/actions.nim index 707a401..03c9f33 100644 --- a/norg/utils/actions.nim +++ b/norg/utils/actions.nim @@ -2,5 +2,5 @@ import osproc proc run_actions*(actions: seq[string]): int {.discardable.} = for action in actions: - echo execCmd(action) + discard execCmd(action) diff --git a/readme.md b/readme.md index 831410f..135a980 100644 --- a/readme.md +++ b/readme.md @@ -1,16 +1,16 @@ # Norg A simple, portable, wrapper for the [borg backup utility](https://www.borgbackup.org) written in Nim + Inspired by [Borgmatic](https://torsion.org/borgmatic) ## Usage Norg uses a `toml` based config file for configuration. An example configuration would look like this: ```toml -source_dirs = [ +source_directories = [ "/home/me/Music", "/home/me/Pictures" ] -encryption_password = "MyReallySecurePassword" [[repositories]] label = "A Repository" path = "/my/backup/location" @@ -19,6 +19,15 @@ path = "/my/backup/location" label = "Another Respository at BorgBase" path = "ssh://1234abcd@1234abcd.repo.borgbase.com/./repo" +[encryption] +encryption_passphrase = "MyReallySecurePassword" + +[actions] +before_actions = ["echo before actions"] +after_actions = ["echo after actions", "echo actions completed"] +before_backup = ["echo before backup", "date"] +after_backup = ["echo after backup","echo backup completed"] + [uptimekuma] base_url = "https://uptime.kuma.url/api/push/1234abcd" states = ["Success","Failure", "Running"] @@ -49,6 +58,20 @@ norg -c myconfig.toml extract pcname-2024-08-18T15:20:17773204 norg -c myconfig.toml extract pcname-2024-08-18T15:20:17773204 --destination /tmp/my_extracted_archive ``` +# Build from Source +Download and build from source +```sh +git clone https://codeberg.org/pswilde/norgbackup +cd norgbackup +nimble install +``` + +or just install directly with `nimble` + +```sh +nimble install https://codeberg.org/pswilde/norgbackup +``` + ## Naming. Why "Norg"? Well, I don't know. I'm a Star Trek fan so obviously I wanted to keep something diff --git a/todo.md b/todo.md index e6990c3..11c8755 100644 --- a/todo.md +++ b/todo.md @@ -1,7 +1,7 @@ # A list of things I'd like to include in Norg - [ ] Backup maintenance i.e. Keep Daily/Weekly/Monthy -- [ ] Pre and post run scripts i.e. to backup a database -- [ ] change encryption_password to encryption_passphrase in config to be more in line with borgbackup +- [x] Pre and post run scripts i.e. to backup a database +- [x] change encryption_password to encryption_passphrase in config to be more in line with borgbackup - [ ] More notifiers (ntfy, healthchecks, etc.) - [ ] Generate config command parameter - [ ] Allow to specify direct repository so mount/extract doesn't fail if multiple are available