Added support for grafana loki
This commit is contained in:
parent
6dca7c1c15
commit
e576403b64
4 changed files with 150 additions and 1 deletions
|
@ -1403,3 +1403,33 @@ properties:
|
||||||
Configuration for a monitoring integration with Crunhub. Create an
|
Configuration for a monitoring integration with Crunhub. Create an
|
||||||
account at https://cronhub.io if you'd like to use this service. See
|
account at https://cronhub.io if you'd like to use this service. See
|
||||||
borgmatic monitoring documentation for details.
|
borgmatic monitoring documentation for details.
|
||||||
|
loki:
|
||||||
|
type: object
|
||||||
|
required: ['url', 'labels']
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
url:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Grafana loki log URL to notify when a backup begins,
|
||||||
|
ends, or fails.
|
||||||
|
example: "http://localhost:3100/loki/api/v1/push"
|
||||||
|
labels:
|
||||||
|
type: object
|
||||||
|
additionalProperties:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Allows setting custom labels for the logging stream. At
|
||||||
|
least one label is required. "__hostname" gets replaced by
|
||||||
|
the machine hostname automatically. "__config" gets replaced
|
||||||
|
by just the name of the configuration file. "__config_path"
|
||||||
|
gets replaced by the full path of the configuration file.
|
||||||
|
example:
|
||||||
|
app: "borgmatic"
|
||||||
|
config: "__config"
|
||||||
|
hostname: "__hostname"
|
||||||
|
description: |
|
||||||
|
Configuration for a monitoring integration with Grafana loki. You
|
||||||
|
can send the logs to a self-hosted instance or create an account at
|
||||||
|
https://grafana.com/auth/sign-up/create-user. See borgmatic
|
||||||
|
monitoring documentation for details.
|
||||||
|
|
|
@ -11,6 +11,7 @@ from borgmatic.hooks import (
|
||||||
pagerduty,
|
pagerduty,
|
||||||
postgresql,
|
postgresql,
|
||||||
sqlite,
|
sqlite,
|
||||||
|
loki,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -26,6 +27,7 @@ HOOK_NAME_TO_MODULE = {
|
||||||
'pagerduty': pagerduty,
|
'pagerduty': pagerduty,
|
||||||
'postgresql_databases': postgresql,
|
'postgresql_databases': postgresql,
|
||||||
'sqlite_databases': sqlite,
|
'sqlite_databases': sqlite,
|
||||||
|
'loki': loki,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
117
borgmatic/hooks/loki.py
Normal file
117
borgmatic/hooks/loki.py
Normal file
|
@ -0,0 +1,117 @@
|
||||||
|
import logging
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import platform
|
||||||
|
from borgmatic.hooks import monitor
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MONITOR_STATE_TO_HEALTHCHECKS = {
|
||||||
|
monitor.State.START: 'Started',
|
||||||
|
monitor.State.FINISH: 'Finished',
|
||||||
|
monitor.State.FAIL: 'Failed',
|
||||||
|
}
|
||||||
|
|
||||||
|
# Threshold at which logs get flushed to loki
|
||||||
|
MAX_BUFFER_LINES = 100
|
||||||
|
|
||||||
|
class loki_log_buffer():
|
||||||
|
'''
|
||||||
|
A log buffer that allows to output the logs as loki requests in json
|
||||||
|
'''
|
||||||
|
def __init__(self, url, dry_run):
|
||||||
|
self.url = url
|
||||||
|
self.dry_run = dry_run
|
||||||
|
self.root = {}
|
||||||
|
self.root["streams"] = [{}]
|
||||||
|
self.root["streams"][0]["stream"] = {}
|
||||||
|
self.root["streams"][0]["values"] = []
|
||||||
|
|
||||||
|
def add_value(self, value):
|
||||||
|
timestamp = str(time.time_ns())
|
||||||
|
self.root["streams"][0]["values"].append((timestamp, value))
|
||||||
|
|
||||||
|
def add_label(self, label, value):
|
||||||
|
self.root["streams"][0]["stream"][label] = value
|
||||||
|
|
||||||
|
def _to_request(self):
|
||||||
|
return json.dumps(self.root)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.root["streams"][0]["values"])
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
if self.dry_run:
|
||||||
|
self.root["streams"][0]["values"] = []
|
||||||
|
return
|
||||||
|
if len(self) == 0:
|
||||||
|
return
|
||||||
|
request_body = self._to_request()
|
||||||
|
self.root["streams"][0]["values"] = []
|
||||||
|
request_header = {"Content-Type": "application/json"}
|
||||||
|
try:
|
||||||
|
r = requests.post(self.url, headers=request_header, data=request_body, timeout=5)
|
||||||
|
r.raise_for_status()
|
||||||
|
except requests.RequestException:
|
||||||
|
logger.warn("Failed to upload logs to loki")
|
||||||
|
|
||||||
|
class loki_log_handeler(logging.Handler):
|
||||||
|
'''
|
||||||
|
A log handler that sends logs to loki
|
||||||
|
'''
|
||||||
|
def __init__(self, url, dry_run):
|
||||||
|
super().__init__()
|
||||||
|
self.buffer = loki_log_buffer(url, dry_run)
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
self.raw(record.getMessage())
|
||||||
|
|
||||||
|
def add_label(self, key, value):
|
||||||
|
self.buffer.add_label(key, value)
|
||||||
|
|
||||||
|
def raw(self, msg):
|
||||||
|
self.buffer.add_value(msg)
|
||||||
|
if len(self.buffer) > MAX_BUFFER_LINES:
|
||||||
|
self.buffer.flush()
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
if len(self.buffer) > 0:
|
||||||
|
self.buffer.flush()
|
||||||
|
|
||||||
|
def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Add a handler to the root logger to regularly send the logs to loki
|
||||||
|
'''
|
||||||
|
url = hook_config.get('url')
|
||||||
|
loki = loki_log_handeler(url, dry_run)
|
||||||
|
for k, v in hook_config.get('labels').items():
|
||||||
|
if v == '__hostname':
|
||||||
|
loki.add_label(k, platform.node())
|
||||||
|
elif v == '__config':
|
||||||
|
loki.add_label(k, config_filename.split('/')[-1])
|
||||||
|
elif v == '__config_path':
|
||||||
|
loki.add_label(k, config_filename)
|
||||||
|
else:
|
||||||
|
loki.add_label(k, v)
|
||||||
|
logging.getLogger().addHandler(loki)
|
||||||
|
|
||||||
|
def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Adds an entry to the loki logger with the current state
|
||||||
|
'''
|
||||||
|
if not dry_run:
|
||||||
|
for handler in tuple(logging.getLogger().handlers):
|
||||||
|
if isinstance(handler, loki_log_handeler):
|
||||||
|
if state in MONITOR_STATE_TO_HEALTHCHECKS.keys():
|
||||||
|
handler.raw(f'{config_filename} {MONITOR_STATE_TO_HEALTHCHECKS[state]} backup')
|
||||||
|
|
||||||
|
def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run):
|
||||||
|
'''
|
||||||
|
Remove the monitor handler that was added to the root logger.
|
||||||
|
'''
|
||||||
|
logger = logging.getLogger()
|
||||||
|
for handler in tuple(logger.handlers):
|
||||||
|
if isinstance(handler, loki_log_handeler):
|
||||||
|
handler.flush()
|
||||||
|
logger.removeHandler(handler)
|
|
@ -1,6 +1,6 @@
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy')
|
MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty', 'ntfy', 'loki')
|
||||||
|
|
||||||
|
|
||||||
class State(Enum):
|
class State(Enum):
|
||||||
|
|
Loading…
Reference in a new issue