From e3d1559cd67b5f76fc83686358e6627f70b5b969 Mon Sep 17 00:00:00 2001 From: Dominika Liberda Date: Thu, 20 May 2021 22:11:48 +0200 Subject: [PATCH] Initial commit --- README.md | 112 ++++++++++++++++ code/func.sh | 63 +++++++++ code/notify.sh | 26 ++++ code/ping.sh | 39 ++++++ code/plot.awk | 240 ++++++++++++++++++++++++++++++++++ code/req.sh | 43 ++++++ config.sh | 10 ++ routes.sh | 14 ++ webroot/index.shs | 8 ++ webroot/uwu.shs | 44 +++++++ workers/every_1min/control | 0 workers/every_1min/worker.sh | 3 + workers/every_30min/control | 0 workers/every_30min/worker.sh | 3 + workers/every_5min/control | 0 workers/every_5min/worker.sh | 3 + workers/notify/control | 0 workers/notify/worker.sh | 2 + 18 files changed, 610 insertions(+) create mode 100644 README.md create mode 100644 code/func.sh create mode 100755 code/notify.sh create mode 100755 code/ping.sh create mode 100644 code/plot.awk create mode 100755 code/req.sh create mode 100644 config.sh create mode 100644 routes.sh create mode 100644 webroot/index.shs create mode 100644 webroot/uwu.shs create mode 100644 workers/every_1min/control create mode 100755 workers/every_1min/worker.sh create mode 100644 workers/every_30min/control create mode 100755 workers/every_30min/worker.sh create mode 100644 workers/every_5min/control create mode 100755 workers/every_5min/worker.sh create mode 100644 workers/notify/control create mode 100755 workers/notify/worker.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..0a4a04c --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# DownTimeRobot + +Checks for downtimes of your services, providing notifications and fancy SVG +graphs. + +## Installing + +``` +git clone https://git.sakamoto.pl/laudompat/http.sh dtr +cd dtr +git clone https://git.sakamoto.pl/domi/dtr app +mkdir -p storage/appconfig storage/data storage/reports +``` + +Afterwards, fill out SMTP settings and `cfg[telegram_bot_token]` if you'd +like to use notifications, and recreate the following file structure under +`storage/appconfig`: + +- every_1min.json, every_5min.json, every_30min.json + +These files control what tests are done, and in what time interval. + +``` +[ + { + "type": "ping", + "label": "Gateway", + "notify": ["mail"], + "data": { + "addr": "10.21.37.1" + } + }, + { + "type": "ping", + "label": "CF 1.1", + "notify": [], + "data": { + "addr": "1.1" + } + }, + { + "type": "request", + "label": "sdomi.pl - body", + "notify": [], + "data": { + "url": "https://sdomi.pl", + "method": "GET", + "match": "body", + "string": "sdomi's webpage" + } + }, + { + "type": "request", + "label": "sdomi.pl - head", + "notify": [], + "data": { + "url": "https://sdomi.pl", + "method": "GET", + "match": "head", + "string": "(S|s)erver: nginx" + } + }, + { + "type": "request", + "label": "sdomi.pl - status", + "notify": ["mail"], + "data": { + "url": "http://sdomi.pl/", + "method": "GET", + "match": "status", + "string": "200" + } + } +] +``` + +- notify.json + +This file controls notification recipient groups. + +``` +[ + { + "type": "mail", + "name": "mail", + "to": [ + "asdf@example.com", + "aoeu@example.com" + ] + }, + { + "type": "telegram", + "name": "group", + "to": [ + 0 + ] + } +] +``` + +- graphs.json + +This file controls what is displayed in the web interface. + +``` +[ + "Gateway", + "CF 1.1", + "sdomi.pl - body", + "sdomi.pl - head" +] +``` diff --git a/code/func.sh b/code/func.sh new file mode 100644 index 0000000..6dfae33 --- /dev/null +++ b/code/func.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +function json_sanitize() { + sed -E 's/"/\\"/g' <<< "$@" | tr -d '\n\r\t' +} + +function json_object() { + local -n ref=$1 + local out='' + out+='{' + for key in ${!ref[@]}; do + out+='"'"$(json_sanitize $key)"'":"'"$(json_sanitize ${ref[$key]})"'",' + done + out="${out::-1}" + out+='}' + echo "$out" +} + +function json_array() { + local -n ref=$1 + local out='' + out+='[' + for i in "${ref[@]}"; do + out+="$i," + done + [[ "$out" != '[' ]] && out="${out::-1}" + out+=']' + echo "$out" +} + +# notify(data, status) +function notify() { + source src/mail.sh + + for j in $1; do + send_data="$(jq -r '.[] | select(.name == "'"$j"'")' < storage/appconfig/notify.json)" + notify_type="$(jq -r '.type' <<< "$send_data")" + if [[ "$notify_type" == "mail" ]]; then + for k in $(jq -r '.to[]' <<< "$send_data"); do + if [[ "$2" == "1" ]]; then + mailsend "$k" "$label is DOWN"\ + "$label went down on $(date)" + elif [[ "$2" == "0" ]]; then + mailsend "$k" "$label is back UP"\ + "$label came back up on $(date)" + fi + done + elif [[ "$notify_type" == "telegram" ]]; then + for k in $(jq -r '.to[]' <<< "$send_data"); do + if [[ "$2" == "1" ]]; then + msg="$label is DOWN" + elif [[ "$2" == "0" ]]; then + msg="$label is back UP" + fi + curl -X POST -H 'Content-Type: application/json'\ + -d '{"chat_id": "'"$k"'", "text": "'"$msg"'", "disable_notification": false}'\ + "https://api.telegram.org/bot${cfg[telegram_bot_token]}/sendMessage" > /dev/null + done + else + echo "not supported (yet)" + fi + done +} diff --git a/code/notify.sh b/code/notify.sh new file mode 100755 index 0000000..9fdb9f0 --- /dev/null +++ b/code/notify.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +for monitor in storage/data/*; do + if [[ "$(tail -n1 < $monitor | jq -r '.status')" == 1 ]]; then + label="$(tail -n1 < $monitor | jq -r '.label')" + label_sha="$(shasum <<< "$label" | cut -c 1-16)" + if [[ ! -f "storage/reports/$label_sha" ]]; then + for file in storage/appconfig/every_*.json; do + data="$(jq -r '.[] | select(.label == "'"$label"'")' $file)" + + if [[ "$data" != '' ]]; then + break + fi + done + + notify="$(jq -r '.notify[]' <<< "$data")" + + if [[ "$notify" != '' ]]; then + source "${cfg[namespace]}/code/func.sh" + + notify "$notify" 1 + touch "storage/reports/$label_sha" + fi + fi + fi +done diff --git a/code/ping.sh b/code/ping.sh new file mode 100755 index 0000000..60e3052 --- /dev/null +++ b/code/ping.sh @@ -0,0 +1,39 @@ +#!/bin/bash +source "${cfg[namespace]}/code/func.sh" + +json="$(jq -r '.[] | select(.type == "ping") | "\(.label)�\(.data.addr)"' < storage/appconfig/$1.json)" + +IFS=$'\n' +for i in $json; do + label="$(awk -F� '{print $1}' <<< "$i")" + label_sha="$(shasum <<< "$label" | cut -c 1-16)" + addr="$(awk -F� '{print $2}' <<< "$i")" + res="$(ping -c 1 -W 1.5 $addr > >(grep icmp_seq | sed -E 's/.*icmp_seq=1//'))" + status=$? + + if [[ "$status" == 0 ]]; then + if [[ "$res" == *"time"* ]]; then + res_parsed="$(grep -Poh "time=.*?ms" <<< "$res" | sed -E 's/time=//g;s/ms//g' | tr -d ' ')" + else + res_parsed='#' + fi + else + res_parsed='#' + fi + + unset json_res + declare -A json_res + json_res[type]="ping" + json_res[label]="$label" + json_res[status]="$status" + json_res[time]="$res_parsed" + json_res[res]="$res" + json_res[date]="$(date "+%T %d.%m.%y")" + + json_object json_res >> "storage/data/$(shasum <<< "$label" | cut -c 1-16)" + if [[ "$status" == 0 && -f "storage/reports/$label_sha" ]]; then + rm "storage/reports/$label_sha" + notify="$(jq -r '.[] | select(.label == "'"$label"'").notify[]' < storage/appconfig/$1.json)" + notify "$notify" 0 + fi +done diff --git a/code/plot.awk b/code/plot.awk new file mode 100644 index 0000000..fb94d3e --- /dev/null +++ b/code/plot.awk @@ -0,0 +1,240 @@ +#!/usr/bin/awk -f +# This program is a copy of guff, a plot device. https://github.com/silentbicycle/guff +# My copy here is written in awk instead of C, has no compelling benefit. +# Public domain. @thingskatedid +# Modified for my purposes by sdomi (ja@sdomi.pl) + +# Run as awk -v x=xyz ... or env variables for stuff? +# Assumptions: the data is evenly spaced along the x-axis + +# TODO: moving average +# TODO: trend lines, or guess at complexities +# TODO: points vs. lines +# TODO: colourblind safe scheme +# TODO: center data around the 0 axis +# TODO: scanning for all float formats input, including -inf, NaN etc +# TODO: guess at whether to use lines or circles, based on delta within a window? + +function hastitle() { + for (i = 1; i <= NF; i++) { + if ($i ~ /[^-0-9.]/) { + return 1 + } + } + + return 0 +} + +function amax(a, i, max) { + max = -1 + + for (i in a) { + if (max == -1 || a[i] > a[max]) { + max = i + } + } + + return max +} + +function normalise( delta) { + for (i = 1; i <= NF; i++) { + max[i] = 0 + min[i] = 0 + + for (j = 1; j <= NR; j++) { + if (a[i, j] != "#") { + if (a[i, j] > max[i]) { + max[i] = a[i, j] + } else + if (a[i, j] < min[i]) { + min[i] = a[i, j] + } + } + } + + delta[i] = max[i] - min[i] + + for (j = 1; j <= NR; j++) { + if (a[i, j] != "#") { + a[i, j] -= min[i] + if (delta[i] > 0) { + a[i, j] /= delta[i] + } + } + } + } + + # TODO: rescale to center around 0 + + # Here the data is squished slightly in descending order of deltas. + # Each column is scaled independently anyway, so they're never to scale. + # The idea here is to help show intutively which are smaller, but without + # actually drawing them to size (since then very small deltas would not be + # visible at all). + + k = 0 + prev = -1 + + while (length(delta) > 0) { + i = amax(delta) + + # Several columns can share the same delta + # Formatting to %.3f here is just for sake of rounding + if (prev != -1 && sprintf("%.3f", prev) != sprintf("%.3f", delta[i])) { + k++ + } + + # +2 to squish things upwards a bit + scale = (NF + 2 - k) / (NF + 2) + + # there's no need to scale by 1 + if (scale != 1) { + for (j = 1; j <= NR; j++) { + if (a[i, j] != "#") { + a[i, j] *= scale + } + } + } + + prev = delta[i] + delete delta[i] + } +} + +# internal coordinates to svg coordinates +function point(x, y) { + x = x * (chart_width - 2 * xmargin) + xmargin + y = (height - 2 * ymargin) - y * (height - 2 * ymargin) + ymargin + + return sprintf("%u,%u", x, y) +} + +function line(i) { + last_was_empty = 0 + printf "" + printf "\n" + last_was_empty = 0 +} + +function circles(i) { + for (j = 1; j <= NR; j++) { + p = point((j - 1) / NR, a[i, j]) + split(p, q, ",") + printf "\n", + q[1], q[2] + } +} + +function legend_text(i, title) { + printf " \n", chart_width + gutter, i * line_height + printf " \n", + -10, -line_height / 2 + 5, color[i], color[i] + printf " %-*s[%.3g, %.3g]\n", + sprintf("fill: %s; font-size: %upx; font-family: mono", fg, font_size), + (title_width > 0) ? title_width + 1 : 0, title, min[i], max[i] + printf " \n" +} + +function display() { + print "" + printf "\n", + "http://www.w3.org/2000/svg", + chart_width + gutter + legend_width, height+line_height + printf "" + title_width = 0 + for (i = 1; i <= NF; i++) { + if (length(title[i]) > title_width) { + title_width = length(title[i]) + } + } + + for (i = 1; i <= NF; i++) { + line(i) + circles(i) + } + + if (length(title)) { + for (i = 1; i <= NF; i++) { + legend_text(i, title[i]) + } + } + printf "###START_DATE###", 5, height+line_height-3 + printf "###END_DATE###", chart_width-158, height+line_height-3 + printf "", height+line_height/2, 10 + printf "", chart_width, height+line_height/2, chart_width, 10 + print "" +} + +NR == 1 { + if (hastitle()) { + for (i = 1; i <= NF; i++) { + title[i] = $i + } + + NR-- + next + } +} + +{ + for (i = 1; i <= NF; i++) { + a[i, NR] = $i + #if (a[i, NR] == "?") { + # printf "%s", a[i, NR] + #} + } +} + +END { + fg = "#eeeeee" + alpha = "ff" + + # Bang Wong's colour-safe palette, https://www.nature.com/articles/nmeth.1618 + # (using just the last five colours) + color[3] = "#CC79A7" + color[1] = "#009E73" + color[2] = "#F0E442" + color[4] = "#0072B2" + color[5] = "#D55E00" + + color[6] = fg + + #if (NF == 1) { + # color[1] = fg + #} + + if (NF > length(color)) { + print "too many fields" >> "/dev/stderr" + exit 1 + } + + chart_width=600 + legend_width=256 + height=120 + xmargin=0 + ymargin=5 + gutter=30 + font_size=15 + line_height=20 + + # the data is scaled 0..1 for our internal coordinate space + normalise() + + display() +} diff --git a/code/req.sh b/code/req.sh new file mode 100755 index 0000000..9bc8508 --- /dev/null +++ b/code/req.sh @@ -0,0 +1,43 @@ +#!/bin/bash +source "${cfg[namespace]}/code/func.sh" + +json="$(jq -r '.[] | select(.type == "request") | "\(.label)�\(.data.url)�\(.data.method)�\(.data.match)�\(.data.string)�\(.data.data)"' < storage/appconfig/$1.json)" + +IFS=$'\n' +for i in $json; do + label="$(awk -F� '{print $1}' <<< "$i")" + label_sha="$(shasum <<< "$label" | cut -c 1-16)" + url="$(awk -F� '{print $2}' <<< "$i")" + method="$(awk -F� '{print $3}' <<< "$i")" + match="$(awk -F� '{print $4}' <<< "$i")" + string="$(awk -F� '{print $5}' <<< "$i")" + data="$(awk -F� '{print $6}' <<< "$i")" + + [[ "$method" == '' ]] && method="GET" + + if [[ "$match" == "body" ]]; then + res="$(curl -A "${cfg[useragent]}" -s -X "$method" "$url" --data "$data" | grep -P "$string")" + status=$? + elif [[ "$match" == "head" ]]; then + res="$(curl -A "${cfg[useragent]}" -sD - -o /dev/null -X "$method" "$url" --data "$data" | grep -P "$string")" + status=$? + elif [[ "$match" == "status" ]]; then + res="$(curl -A "${cfg[useragent]}" -sD - -o /dev/null -X "$method" "$url" --data "$data" | grep -P "HTTP.*? $string")" + status=$? + fi + + unset json_res + declare -A json_res + json_res[type]="req" + json_res[label]="$label" + json_res[status]="$status" + json_res[res]="$res" + json_res[date]="$(date "+%T %d.%m.%y")" + + json_object json_res >> "storage/data/$label_sha" + if [[ "$status" == 0 && -f "storage/reports/$label_sha" ]]; then + rm "storage/reports/$label_sha" + notify="$(jq -r '.[] | select(.label == "'"$label"'").notify[]' < storage/appconfig/$1.json)" + notify "$notify" 0 + fi +done diff --git a/config.sh b/config.sh new file mode 100644 index 0000000..c9c1a89 --- /dev/null +++ b/config.sh @@ -0,0 +1,10 @@ +## app config +## your application-specific config goes here! + +cfg[useragent]="dtr/1.0 (https://git.sakamoto.pl/domi/dtr)" +cfg[telegram_bot_token]="" + +worker_add every_1min 60 +worker_add every_5min 300 +worker_add every_30min 1800 +worker_add notify 60 diff --git a/routes.sh b/routes.sh new file mode 100644 index 0000000..c8aad36 --- /dev/null +++ b/routes.sh @@ -0,0 +1,14 @@ +## routes - application-specific routes +## +## HTTP.sh supports both serving files using a directory structure (webroot), +## and using routes. The latter may come in handy if you want to create nicer +## paths, e.g. +## +## (webroot) https://example.com/profile.shs?name=ptrcnull +## ... may become ... +## (routes) https://example.com/profile/ptrcnull +## +## To set up routes, define rules in this file (see below for examples) + +# router "/test" "app/views/test.shs" +# router "/profile/:user" "app/views/user.shs" diff --git a/webroot/index.shs b/webroot/index.shs new file mode 100644 index 0000000..1fbea57 --- /dev/null +++ b/webroot/index.shs @@ -0,0 +1,8 @@ +#!/bin/bash + +source "templates/head.sh" +IFS=$'\n' +for i in $(jq -r '.[]' < storage/appconfig/graphs.json); do + source "${cfg[namespace]}/webroot/uwu.shs" "$(shasum <<< $i | cut -c 1-16)" +done +#echo '' diff --git a/webroot/uwu.shs b/webroot/uwu.shs new file mode 100644 index 0000000..80fd63a --- /dev/null +++ b/webroot/uwu.shs @@ -0,0 +1,44 @@ +#!/bin/bash +lim=120 + +if [[ "$1" != '' ]]; then + data="storage/data/$(basename $1)" +elif [[ "${get_data[id]}" != '' ]]; then + data="storage/data/$(basename ${get_data[id]})" + header "Content-Type: image/svg+xml" +else + exit 0 +fi + +label="$(jq -r '.label' < $data | head -n 1 | tr ' ' '_')" +start_date="$(jq -r '.date' < $data | tail -n $lim | head -n1)" +end_date="$(jq -r '.date' < $data | tail -n1)" +tmp="$(mktemp)" + +if [[ "$(jq -r '.type' < $data | head -n 1)" == "ping" ]]; then + timedata="$(jq -r '.time' < $data | tail -n $lim)" + + while read line; do + echo -n 's@'"'"'>'"$line"' $tmp + +elif [[ "$(jq -r '.type' < $data | head -n 1)" == "req" ]]; then + timedata="$(jq -r '.status' < $data | tail -n $lim)" + a="" + for i in $timedata; do + if [[ "$i" -gt 0 ]]; then + a="${a}0#" + elif [[ "$i" == "0" ]]; then + a="${a}1#" + fi + done + timedata="$(tr '#' '\n' <<< "$a")" + while read line; do + echo -n 's@'"'"'>'"$line"' $tmp +fi + +svg="$(echo -e "$label\n$timedata" | awk -f "${cfg[namespace]}/code/plot.awk")" +tr -d '\n' <<< "$svg" | sed -f $tmp | sed 's@###START_DATE###@'"$start_date"'@;s@###END_DATE###@'"$end_date"'@' + +rm $tmp diff --git a/workers/every_1min/control b/workers/every_1min/control new file mode 100644 index 0000000..e69de29 diff --git a/workers/every_1min/worker.sh b/workers/every_1min/worker.sh new file mode 100755 index 0000000..9d04f82 --- /dev/null +++ b/workers/every_1min/worker.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source "${cfg[namespace]}/code/ping.sh" every_1min +source "${cfg[namespace]}/code/req.sh" every_1min diff --git a/workers/every_30min/control b/workers/every_30min/control new file mode 100644 index 0000000..e69de29 diff --git a/workers/every_30min/worker.sh b/workers/every_30min/worker.sh new file mode 100755 index 0000000..6ca2238 --- /dev/null +++ b/workers/every_30min/worker.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source "${cfg[namespace]}/code/ping.sh" every_30min +source "${cfg[namespace]}/code/req.sh" every_30min diff --git a/workers/every_5min/control b/workers/every_5min/control new file mode 100644 index 0000000..e69de29 diff --git a/workers/every_5min/worker.sh b/workers/every_5min/worker.sh new file mode 100755 index 0000000..f397fb4 --- /dev/null +++ b/workers/every_5min/worker.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source "${cfg[namespace]}/code/ping.sh" every_5min +source "${cfg[namespace]}/code/req.sh" every_5min diff --git a/workers/notify/control b/workers/notify/control new file mode 100644 index 0000000..e69de29 diff --git a/workers/notify/worker.sh b/workers/notify/worker.sh new file mode 100755 index 0000000..88db57e --- /dev/null +++ b/workers/notify/worker.sh @@ -0,0 +1,2 @@ +#!/bin/bash +source "${cfg[namespace]}/code/notify.sh"