Added support for grafana loki

This commit is contained in:
Tobias Hodapp 2023-08-22 03:13:39 +02:00
parent 6dca7c1c15
commit e576403b64
4 changed files with 150 additions and 1 deletions

View file

@ -1403,3 +1403,33 @@ properties:
Configuration for a monitoring integration with Crunhub. Create an
account at https://cronhub.io if you'd like to use this service. See
borgmatic monitoring documentation for details.
loki:
type: object
required: ['url', 'labels']
additionalProperties: false
properties:
url:
type: string
description: |
Grafana loki log URL to notify when a backup begins,
ends, or fails.
example: "http://localhost:3100/loki/api/v1/push"
labels:
type: object
additionalProperties:
type: string
description: |
Allows setting custom labels for the logging stream. At
least one label is required. "__hostname" gets replaced by
the machine hostname automatically. "__config" gets replaced
by just the name of the configuration file. "__config_path"
gets replaced by the full path of the configuration file.
example:
app: "borgmatic"
config: "__config"
hostname: "__hostname"
description: |
Configuration for a monitoring integration with Grafana loki. You
can send the logs to a self-hosted instance or create an account at
https://grafana.com/auth/sign-up/create-user. See borgmatic
monitoring documentation for details.

View file

@ -11,6 +11,7 @@ from borgmatic.hooks import (
pagerduty,
postgresql,
sqlite,
loki,
)
logger = logging.getLogger(__name__)
@ -26,6 +27,7 @@ HOOK_NAME_TO_MODULE = {
'pagerduty': pagerduty,
'postgresql_databases': postgresql,
'sqlite_databases': sqlite,
'loki': loki,
}

117
borgmatic/hooks/loki.py Normal file
View file

@ -0,0 +1,117 @@
import logging
import requests
import json
import time
import platform
from borgmatic.hooks import monitor
logger = logging.getLogger(__name__)
MONITOR_STATE_TO_HEALTHCHECKS = {
monitor.State.START: 'Started',
monitor.State.FINISH: 'Finished',
monitor.State.FAIL: 'Failed',
}
# Threshold at which logs get flushed to loki
MAX_BUFFER_LINES = 100
class loki_log_buffer():
'''
A log buffer that allows to output the logs as loki requests in json
'''
def __init__(self, url, dry_run):
self.url = url
self.dry_run = dry_run
self.root = {}
self.root["streams"] = [{}]
self.root["streams"][0]["stream"] = {}
self.root["streams"][0]["values"] = []
def add_value(self, value):
timestamp = str(time.time_ns())
self.root["streams"][0]["values"].append((timestamp, value))
def add_label(self, label, value):
self.root["streams"][0]["stream"][label] = value
def _to_request(self):
return json.dumps(self.root)
def __len__(self):
return len(self.root["streams"][0]["values"])
def flush(self):
if self.dry_run:
self.root["streams"][0]["values"] = []
return
if len(self) == 0:
return
request_body = self._to_request()
self.root["streams"][0]["values"] = []
request_header = {"Content-Type": "application/json"}
try:
r = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
r.raise_for_status()
except requests.RequestException:
logger.warn("Failed to upload logs to loki")
class loki_log_handeler(logging.Handler):
'''
A log handler that sends logs to loki
'''
def __init__(self, url, dry_run):
super().__init__()
self.buffer = loki_log_buffer(url, dry_run)
def emit(self, record):
self.raw(record.getMessage())
def add_label(self, key, value):
self.buffer.add_label(key, value)
def raw(self, msg):
self.buffer.add_value(msg)
if len(self.buffer) > MAX_BUFFER_LINES:
self.buffer.flush()
def flush(self):
if len(self.buffer) > 0:
self.buffer.flush()
def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
'''
Add a handler to the root logger to regularly send the logs to loki
'''
url = hook_config.get('url')
loki = loki_log_handeler(url, dry_run)
for k, v in hook_config.get('labels').items():
if v == '__hostname':
loki.add_label(k, platform.node())
elif v == '__config':
loki.add_label(k, config_filename.split('/')[-1])
elif v == '__config_path':
loki.add_label(k, config_filename)
else:
loki.add_label(k, v)
logging.getLogger().addHandler(loki)
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
'''
Adds an entry to the loki logger with the current state
'''
if not dry_run:
for handler in tuple(logging.getLogger().handlers):
if isinstance(handler, loki_log_handeler):
if state in MONITOR_STATE_TO_HEALTHCHECKS.keys():
handler.raw(f'{config_filename} {MONITOR_STATE_TO_HEALTHCHECKS[state]} backup')
def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
'''
Remove the monitor handler that was added to the root logger.
'''
logger = logging.getLogger()
for handler in tuple(logger.handlers):
if isinstance(handler, loki_log_handeler):
handler.flush()
logger.removeHandler(handler)

View file

@ -1,6 +1,6 @@
from enum import Enum
MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy')
MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy', 'loki')
class State(Enum):