Commit e3d1559c authored by Dominika Liberda's avatar Dominika Liberda 💽
Browse files

Initial commit

parents
# DownTimeRobot
Checks for downtimes of your services, providing notifications and fancy SVG
graphs.
## Installing
```
git clone https://git.sakamoto.pl/laudompat/http.sh dtr
cd dtr
git clone https://git.sakamoto.pl/domi/dtr app
mkdir -p storage/appconfig storage/data storage/reports
```
Afterwards, fill out SMTP settings and `cfg[telegram_bot_token]` if you'd
like to use notifications, and recreate the following file structure under
`storage/appconfig`:
- every_1min.json, every_5min.json, every_30min.json
These files control what tests are done, and in what time interval.
```
[
{
"type": "ping",
"label": "Gateway",
"notify": ["mail"],
"data": {
"addr": "10.21.37.1"
}
},
{
"type": "ping",
"label": "CF 1.1",
"notify": [],
"data": {
"addr": "1.1"
}
},
{
"type": "request",
"label": "sdomi.pl - body",
"notify": [],
"data": {
"url": "https://sdomi.pl",
"method": "GET",
"match": "body",
"string": "sdomi's webpage"
}
},
{
"type": "request",
"label": "sdomi.pl - head",
"notify": [],
"data": {
"url": "https://sdomi.pl",
"method": "GET",
"match": "head",
"string": "(S|s)erver: nginx"
}
},
{
"type": "request",
"label": "sdomi.pl - status",
"notify": ["mail"],
"data": {
"url": "http://sdomi.pl/",
"method": "GET",
"match": "status",
"string": "200"
}
}
]
```
- notify.json
This file controls notification recipient groups.
```
[
{
"type": "mail",
"name": "mail",
"to": [
"asdf@example.com",
"aoeu@example.com"
]
},
{
"type": "telegram",
"name": "group",
"to": [
0
]
}
]
```
- graphs.json
This file controls what is displayed in the web interface.
```
[
"Gateway",
"CF 1.1",
"sdomi.pl - body",
"sdomi.pl - head"
]
```
#!/bin/bash
function json_sanitize() {
sed -E 's/"/\\"/g' <<< "$@" | tr -d '\n\r\t'
}
function json_object() {
local -n ref=$1
local out=''
out+='{'
for key in ${!ref[@]}; do
out+='"'"$(json_sanitize $key)"'":"'"$(json_sanitize ${ref[$key]})"'",'
done
out="${out::-1}"
out+='}'
echo "$out"
}
function json_array() {
local -n ref=$1
local out=''
out+='['
for i in "${ref[@]}"; do
out+="$i,"
done
[[ "$out" != '[' ]] && out="${out::-1}"
out+=']'
echo "$out"
}
# notify(data, status)
function notify() {
source src/mail.sh
for j in $1; do
send_data="$(jq -r '.[] | select(.name == "'"$j"'")' < storage/appconfig/notify.json)"
notify_type="$(jq -r '.type' <<< "$send_data")"
if [[ "$notify_type" == "mail" ]]; then
for k in $(jq -r '.to[]' <<< "$send_data"); do
if [[ "$2" == "1" ]]; then
mailsend "$k" "$label is DOWN"\
"$label went down on $(date)"
elif [[ "$2" == "0" ]]; then
mailsend "$k" "$label is back UP"\
"$label came back up on $(date)"
fi
done
elif [[ "$notify_type" == "telegram" ]]; then
for k in $(jq -r '.to[]' <<< "$send_data"); do
if [[ "$2" == "1" ]]; then
msg="$label is DOWN"
elif [[ "$2" == "0" ]]; then
msg="$label is back UP"
fi
curl -X POST -H 'Content-Type: application/json'\
-d '{"chat_id": "'"$k"'", "text": "'"$msg"'", "disable_notification": false}'\
"https://api.telegram.org/bot${cfg[telegram_bot_token]}/sendMessage" > /dev/null
done
else
echo "not supported (yet)"
fi
done
}
#!/bin/bash
for monitor in storage/data/*; do
if [[ "$(tail -n1 < $monitor | jq -r '.status')" == 1 ]]; then
label="$(tail -n1 < $monitor | jq -r '.label')"
label_sha="$(shasum <<< "$label" | cut -c 1-16)"
if [[ ! -f "storage/reports/$label_sha" ]]; then
for file in storage/appconfig/every_*.json; do
data="$(jq -r '.[] | select(.label == "'"$label"'")' $file)"
if [[ "$data" != '' ]]; then
break
fi
done
notify="$(jq -r '.notify[]' <<< "$data")"
if [[ "$notify" != '' ]]; then
source "${cfg[namespace]}/code/func.sh"
notify "$notify" 1
touch "storage/reports/$label_sha"
fi
fi
fi
done
#!/bin/bash
source "${cfg[namespace]}/code/func.sh"
json="$(jq -r '.[] | select(.type == "ping") | "\(.label)�\(.data.addr)"' < storage/appconfig/$1.json)"
IFS=$'\n'
for i in $json; do
label="$(awk -F'{print $1}' <<< "$i")"
label_sha="$(shasum <<< "$label" | cut -c 1-16)"
addr="$(awk -F'{print $2}' <<< "$i")"
res="$(ping -c 1 -W 1.5 $addr > >(grep icmp_seq | sed -E 's/.*icmp_seq=1//'))"
status=$?
if [[ "$status" == 0 ]]; then
if [[ "$res" == *"time"* ]]; then
res_parsed="$(grep -Poh "time=.*?ms" <<< "$res" | sed -E 's/time=//g;s/ms//g' | tr -d ' ')"
else
res_parsed='#'
fi
else
res_parsed='#'
fi
unset json_res
declare -A json_res
json_res[type]="ping"
json_res[label]="$label"
json_res[status]="$status"
json_res[time]="$res_parsed"
json_res[res]="$res"
json_res[date]="$(date "+%T %d.%m.%y")"
json_object json_res >> "storage/data/$(shasum <<< "$label" | cut -c 1-16)"
if [[ "$status" == 0 && -f "storage/reports/$label_sha" ]]; then
rm "storage/reports/$label_sha"
notify="$(jq -r '.[] | select(.label == "'"$label"'").notify[]' < storage/appconfig/$1.json)"
notify "$notify" 0
fi
done
#!/usr/bin/awk -f
# This program is a copy of guff, a plot device. https://github.com/silentbicycle/guff
# My copy here is written in awk instead of C, has no compelling benefit.
# Public domain. @thingskatedid
# Modified for my purposes by sdomi (ja@sdomi.pl)
# Run as awk -v x=xyz ... or env variables for stuff?
# Assumptions: the data is evenly spaced along the x-axis
# TODO: moving average
# TODO: trend lines, or guess at complexities
# TODO: points vs. lines
# TODO: colourblind safe scheme
# TODO: center data around the 0 axis
# TODO: scanning for all float formats input, including -inf, NaN etc
# TODO: guess at whether to use lines or circles, based on delta within a window?
function hastitle() {
for (i = 1; i <= NF; i++) {
if ($i ~ /[^-0-9.]/) {
return 1
}
}
return 0
}
function amax(a, i, max) {
max = -1
for (i in a) {
if (max == -1 || a[i] > a[max]) {
max = i
}
}
return max
}
function normalise( delta) {
for (i = 1; i <= NF; i++) {
max[i] = 0
min[i] = 0
for (j = 1; j <= NR; j++) {
if (a[i, j] != "#") {
if (a[i, j] > max[i]) {
max[i] = a[i, j]
} else
if (a[i, j] < min[i]) {
min[i] = a[i, j]
}
}
}
delta[i] = max[i] - min[i]
for (j = 1; j <= NR; j++) {
if (a[i, j] != "#") {
a[i, j] -= min[i]
if (delta[i] > 0) {
a[i, j] /= delta[i]
}
}
}
}
# TODO: rescale to center around 0
# Here the data is squished slightly in descending order of deltas.
# Each column is scaled independently anyway, so they're never to scale.
# The idea here is to help show intutively which are smaller, but without
# actually drawing them to size (since then very small deltas would not be
# visible at all).
k = 0
prev = -1
while (length(delta) > 0) {
i = amax(delta)
# Several columns can share the same delta
# Formatting to %.3f here is just for sake of rounding
if (prev != -1 && sprintf("%.3f", prev) != sprintf("%.3f", delta[i])) {
k++
}
# +2 to squish things upwards a bit
scale = (NF + 2 - k) / (NF + 2)
# there's no need to scale by 1
if (scale != 1) {
for (j = 1; j <= NR; j++) {
if (a[i, j] != "#") {
a[i, j] *= scale
}
}
}
prev = delta[i]
delete delta[i]
}
}
# internal coordinates to svg coordinates
function point(x, y) {
x = x * (chart_width - 2 * xmargin) + xmargin
y = (height - 2 * ymargin) - y * (height - 2 * ymargin) + ymargin
return sprintf("%u,%u", x, y)
}
function line(i) {
last_was_empty = 0
printf "<polyline stroke='%s%s' stroke-width='1.5' fill='#0000'", color[i], alpha
printf " points='"
for (j = 1; j <= NR; j++) {
if (a[i, j] == "#") {
if(last_was_empty == 0) {
printf "'/>"
printf "<polyline stroke='%s%s' stroke-width='1.5' fill='#0000'", color[i], alpha
printf " points='"
last_was_empty = 1
}
} else {
printf "%s ", point((j - 1) / NR, a[i, j])
last_was_empty = 0
}
}
printf "'/>\n"
last_was_empty = 0
}
function circles(i) {
for (j = 1; j <= NR; j++) {
p = point((j - 1) / NR, a[i, j])
split(p, q, ",")
printf "<circle cx='%u' cy='%u' r='5'></circle>\n",
q[1], q[2]
}
}
function legend_text(i, title) {
printf " <g transform='translate(%u %u)'>\n", chart_width + gutter, i * line_height
printf " <circle cx='%d' cy='%d' r='3.5' style='fill:%s;' stroke='%s'/>\n",
-10, -line_height / 2 + 5, color[i], color[i]
printf " <text style='%s' xml:space='preserve'>%-*s[%.3g, %.3g]</text>\n",
sprintf("fill: %s; font-size: %upx; font-family: mono", fg, font_size),
(title_width > 0) ? title_width + 1 : 0, title, min[i], max[i]
printf " </g>\n"
}
function display() {
print "<?xml version='1.0'?>"
printf "<svg xmlns='%s' width='%u' height='%u' version='1.1'>\n",
"http://www.w3.org/2000/svg",
chart_width + gutter + legend_width, height+line_height
printf "<style>circle {fill:#0000;} svg {background-color: #222;} text {fill: #eeeeee; font-size: 15px; font-family: mono}</style>"
title_width = 0
for (i = 1; i <= NF; i++) {
if (length(title[i]) > title_width) {
title_width = length(title[i])
}
}
for (i = 1; i <= NF; i++) {
line(i)
circles(i)
}
if (length(title)) {
for (i = 1; i <= NF; i++) {
legend_text(i, title[i])
}
}
printf "<g transform='translate(%u %u)'><text>###START_DATE###</text></g>", 5, height+line_height-3
printf "<g transform='translate(%u %u)'><text>###END_DATE###</text></g>", chart_width-158, height+line_height-3
printf "<polyline stroke='#666f' stroke-width='1.5' fill='#0000' points='1,%s 1,%s'/>", height+line_height/2, 10
printf "<polyline stroke='#666f' stroke-width='1.5' fill='#0000' points='%s,%s %s,%s'/>", chart_width, height+line_height/2, chart_width, 10
print "</svg>"
}
NR == 1 {
if (hastitle()) {
for (i = 1; i <= NF; i++) {
title[i] = $i
}
NR--
next
}
}
{
for (i = 1; i <= NF; i++) {
a[i, NR] = $i
#if (a[i, NR] == "?") {
# printf "%s", a[i, NR]
#}
}
}
END {
fg = "#eeeeee"
alpha = "ff"
# Bang Wong's colour-safe palette, https://www.nature.com/articles/nmeth.1618
# (using just the last five colours)
color[3] = "#CC79A7"
color[1] = "#009E73"
color[2] = "#F0E442"
color[4] = "#0072B2"
color[5] = "#D55E00"
color[6] = fg
#if (NF == 1) {
# color[1] = fg
#}
if (NF > length(color)) {
print "too many fields" >> "/dev/stderr"
exit 1
}
chart_width=600
legend_width=256
height=120
xmargin=0
ymargin=5
gutter=30
font_size=15
line_height=20
# the data is scaled 0..1 for our internal coordinate space
normalise()
display()
}
#!/bin/bash
source "${cfg[namespace]}/code/func.sh"
json="$(jq -r '.[] | select(.type == "request") | "\(.label)�\(.data.url)�\(.data.method)�\(.data.match)�\(.data.string)�\(.data.data)"' < storage/appconfig/$1.json)"
IFS=$'\n'
for i in $json; do
label="$(awk -F'{print $1}' <<< "$i")"
label_sha="$(shasum <<< "$label" | cut -c 1-16)"
url="$(awk -F'{print $2}' <<< "$i")"
method="$(awk -F'{print $3}' <<< "$i")"
match="$(awk -F'{print $4}' <<< "$i")"
string="$(awk -F'{print $5}' <<< "$i")"
data="$(awk -F'{print $6}' <<< "$i")"
[[ "$method" == '' ]] && method="GET"
if [[ "$match" == "body" ]]; then
res="$(curl -A "${cfg[useragent]}" -s -X "$method" "$url" --data "$data" | grep -P "$string")"
status=$?
elif [[ "$match" == "head" ]]; then
res="$(curl -A "${cfg[useragent]}" -sD - -o /dev/null -X "$method" "$url" --data "$data" | grep -P "$string")"
status=$?
elif [[ "$match" == "status" ]]; then
res="$(curl -A "${cfg[useragent]}" -sD - -o /dev/null -X "$method" "$url" --data "$data" | grep -P "HTTP.*? $string")"
status=$?
fi
unset json_res
declare -A json_res
json_res[type]="req"
json_res[label]="$label"
json_res[status]="$status"
json_res[res]="$res"
json_res[date]="$(date "+%T %d.%m.%y")"
json_object json_res >> "storage/data/$label_sha"
if [[ "$status" == 0 && -f "storage/reports/$label_sha" ]]; then
rm "storage/reports/$label_sha"
notify="$(jq -r '.[] | select(.label == "'"$label"'").notify[]' < storage/appconfig/$1.json)"
notify "$notify" 0
fi
done
## app config
## your application-specific config goes here!
cfg[useragent]="dtr/1.0 (https://git.sakamoto.pl/domi/dtr)"
cfg[telegram_bot_token]=""
worker_add every_1min 60
worker_add every_5min 300
worker_add every_30min 1800
worker_add notify 60
## routes - application-specific routes
##
## HTTP.sh supports both serving files using a directory structure (webroot),
## and using routes. The latter may come in handy if you want to create nicer
## paths, e.g.
##
## (webroot) https://example.com/profile.shs?name=ptrcnull
## ... may become ...
## (routes) https://example.com/profile/ptrcnull
##
## To set up routes, define rules in this file (see below for examples)
# router "/test" "app/views/test.shs"
# router "/profile/:user" "app/views/user.shs"
#!/bin/bash
source "templates/head.sh"
IFS=$'\n'
for i in $(jq -r '.[]' < storage/appconfig/graphs.json); do
source "${cfg[namespace]}/webroot/uwu.shs" "$(shasum <<< $i | cut -c 1-16)"
done
#echo '<img src="/uwu.shs">'
#!/bin/bash
lim=120
if [[ "$1" != '' ]]; then
data="storage/data/$(basename $1)"
elif [[ "${get_data[id]}" != '' ]]; then
data="storage/data/$(basename ${get_data[id]})"
header "Content-Type: image/svg+xml"
else
exit 0
fi
label="$(jq -r '.label' < $data | head -n 1 | tr ' ' '_')"
start_date="$(jq -r '.date' < $data | tail -n $lim | head -n1)"
end_date="$(jq -r '.date' < $data | tail -n1)"
tmp="$(mktemp)"
if [[ "$(jq -r '.type' < $data | head -n 1)" == "ping" ]]; then
timedata="$(jq -r '.time' < $data | tail -n $lim)"
while read line; do
echo -n 's@'"'"'></circle@'"'"'><title>'"$line"'</title></circle@;'
done <<< "$(jq -r '"Name: \(.label)\\\\nPing time: \(.time)ms\\\\nDate: \(.date)"' < "$data" | tail -n $lim)" > $tmp
elif [[ "$(jq -r '.type' < $data | head -n 1)" == "req" ]]; then
timedata="$(jq -r '.status' < $data | tail -n $lim)"
a=""
for i in $timedata; do
if [[ "$i" -gt 0 ]]; then
a="${a}0#"
elif [[ "$i" == "0" ]]; then
a="${a}1#"
fi
done
timedata="$(tr '#' '\n' <<< "$a")"
while read line; do
echo -n 's@'"'"'></circle@'"'"'><title>'"$line"'</title></circle@;'
done <<< "$(jq -r '"Name: \(.label)\\\\nDate: \(.date)"' < "$data" | tail -n $lim)" > $tmp
fi
svg="$(echo -e "$label\n$timedata" | awk -f "${cfg[namespace]}/code/plot.awk")"
tr -d '\n' <<< "$svg" | sed -f $tmp | sed 's@###START_DATE###@'"$start_date"'@;s@###END_DATE###@'"$end_date"'@'
rm $tmp
#!/bin/bash
source "${cfg[namespace]}/code/ping.sh" every_1min
source "${cfg[namespace]}/code/req.sh" every_1min
#!/bin/bash
source "${cfg[namespace]}/code/ping.sh" every_30min
source "${cfg[namespace]}/code/req.sh" every_30min