Additional alerts and notifications in Netdata

home/.chezmoi.yaml.tmpl

@@ -12,8 +12,13 @@
 {{- $locale := (output "echo" "$LANG") }}
 {{- $name := (default "Brian Zalewski" (env "FULL_NAME")) -}}
 {{- $restricted := (default false (env "WORK_ENVIRONMENT")) -}}
+{{- $slack_webhook_url := (default "" (env "SLACK_WEBHOOK_URL")) -}}
+{{- $slack_netdata_alarms_channel := (default "" (env "SLACK_NETDATA_ALARMS_CHANNEL")) -}}
 {{- $snapcraftEmail := (default "brian@megabyte.space" (env "SNAPCRAFT_EMAIL"))}}
 {{- $surgeshUsername := (default "brian@megabyte.space" (env "SURGESH_USERNAME")) -}}
+{{- $syslog_netdata_enable := (default false (env "SYSLOG_NETDATA_ENABLE")) -}}
+{{- $syslog_netdata_facility := (default "local6" (env "SYSLOG_NETDATA_FACILITY")) -}}
+{{- $syslog_netdata_recipients := (default "" (env "SYSLOG_NETDATA_RECIPIENTS")) -}}
 {{- $timezone := (default "America/New_York" (env "TIMEZONE")) -}}
 {{- $toolchains := list "CLI-Extras" "Docker" "Go" "Kubernetes" "Web-Development" -}}
 {{- $work := (default false (env "WORK_ENVIRONMENT")) -}}
@@ -159,7 +164,14 @@ data:
         - 192.168.1.0/24
     qubes: {{ ne (stat (joinPath "usr" "bin" "qubes-session")) false }}
     restricted: {{ $restricted }}
+    slack:
+      webhook_url: "{{ $slack_webhook_url }}"
+      netdata_alarms_channel: "{{ $slack_netdata_alarms_channel }}"
     softwareGroup: "{{ $softwareGroup }}"
+    syslog:
+      config: {{ $syslog_netdata_enable }}
+      netdata_facility: "{{ $syslog_netdata_facility }}"
+      netdata_recipients: "{{ $syslog_netdata_recipients }}"
     type: "{{ $chassisType }}"
     work: {{ $work }}
   toolchains:

home/.chezmoiscripts/universal/run_onchange_after_57-netdata.sh.tmpl

@@ -1,3 +1,63 @@
+{{- if ne .host.distro.family "windows" -}}
+#!/usr/bin/env bash
+# @file Netdata
+# @brief Configures Netdata alerts and notifications
+# @description
+#     This script installs additional alerts and enables notifications if Netdata is installed. Email notifications are configured
+#     using the provided email address. Slack notifications are configured if `SLACK_WEBHOOK_URL` and `SLACK_NETDATA_ALARMS_CHANNEL`
+#     are provided. Syslog integration is configured if `SYSLOG_NETDATA_ENABLE` and `SYSLOG_NETDATA_RECIPIENTS` are provided.
+#     If the OS is Debian based, Netdata shows the number of CVEs in currently installed packages.
+
+{{ includeTemplate "universal/profile" }}
+{{ includeTemplate "universal/logg" }}
+
+### Install additional alerts and enable notifications
+if command -v netdata > /dev/null; then
+    # Copy the additional alert definitions
+    logg info 'Copying $HOME/.config/netdata/health.d/ to /etc/netdata/'
+    sudo cp -rf "$HOME/.config/netdata/health.d/" /etc/netdata/
+
+    logg info 'Copying $HOME/.config/netdata/health_alarm_notify.conf to /usr/lib/netdata/conf.d/health_alarm_notify.conf'
+    sudo cp -f /usr/lib/netdata/conf.d/health_alarm_notify.conf /usr/lib/netdata/conf.d/health_alarm_notify.conf.bak
+    sudo cp -f "$HOME/.config/netdata/health_alarm_notify.conf" /usr/lib/netdata/conf.d/health_alarm_notify.conf
+else
+    logg warn '`netdata` is not available in the PATH or is not installed'
+fi
+
+### Configure Netdata to gather information about CVEs in the installed packages
+if command -v debsecan > /dev/null; then
+
+    # Installing the script to generate report on CVEs in installed packages
+    logg info 'Installing script to generate on CVEs in installed packages'
+    sudo curl -sSL https://raw.githubusercontent.com/nodiscc/netdata-debsecan/master/usr_local_bin_debsecan-by-type -o /usr/local/bin/debsecan-by-type
+
+    # Generate initial debsecan reports in /var/log/debsecan/
+    logg info 'Generating initial debsecan reports in /var/log/debsecan/'
+    /usr/local/bin/debsecan-by-type
+
+    # Configure dpkg to refresh the file after each run
+    logg info 'Configuring dpkg to refresh the file after each run'
+    sudo curl -sSL https://github.com/nodiscc/netdata-debsecan/raw/master/etc_apt_apt.conf.d_99debsecan -o /etc/apt/apt.conf.d/99debsecan
+
+    # Add a cron job to refresh the file every hour
+    logg info 'Adding a cron job to refresh the file every hour'
+    sudo curl -sSL https://github.com/nodiscc/netdata-debsecan/raw/master/etc_cron.d_debsecan -o /etc/cron.d/debsecan
+
+    # Install the module/configuration file
+    logg info 'Installing the module and configuration file
+    sudo curl -sSL https://github.com/nodiscc/netdata-debsecan/raw/master/debsecan.chart.py -o /usr/libexec/netdata/python.d/
+    sudo curl -sSL https://github.com/nodiscc/netdata-debsecan/raw/master/debsecan.conf -o /etc/netdata/python.d/
+
+    # Restart Netdata service
+    logg info 'Restarting netdata service'
+    sudo systemctl restart netdata
+else
+    logg warn '`debsecan` is not available in the PATH or is not installed'
+fi
+
+
+{{ end -}}
+
 {{- if and (ne .host.distro.family "windows") (stat (joinPath .host.home ".config" "age" "chezmoi.txt")) (or (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "NETDATA_TOKEN")) (env "NETDATA_TOKEN")) (or (stat (joinPath .chezmoi.sourceDir ".chezmoitemplates" "secrets" "NETDATA_ROOM")) (env "NETDATA_ROOM")) -}}
 #!/usr/bin/env bash
 # @file Netdata

home/dot_config/netdata/health.d/apps.conf

@@ -0,0 +1,26 @@
+alarm: apps_cpu_usage
+on: apps.cpu
+lookup: average -1m percentage *
+units: %
+every: 5m
+warn: $this > 50
+crit: $this > 70
+info: CPU utilization of Applications
+
+alarm: apps_ram_usage
+on: apps.mem
+lookup: average -1m percentage *
+units: %
+every: 5m
+warn: $this > 65
+crit: $this > 80
+info: RAM utilization of Applications
+
+alarm: apps_swap_usage
+on: apps.swap
+lookup: average -1m percentage *
+units: %
+every: 5m
+warn: $this > 50
+crit: $this > 70
+info: Swap utilization of Applications

home/dot_config/netdata/health.d/network.conf

@@ -3,4 +3,4 @@ on: net.drops
 lookup: sum -30m unaligned absolute
 every: 10s
 crit: $this > 0
-info: Dropper network packets
+info: Dropped network packets

software.yml

@@ -1749,6 +1749,15 @@ softwarePackages:
     _name: Debloat Windows
     ansible: professormanhattan.debloat
     _type: cli
+  debsecan:
+    _bin: null
+    _desc: `debsecan` analyzes the list of installed packages on the current host and reports vulnerabilities found on the system.
+    _docs: null
+    _github: null
+    _home: null
+    _name: Debian Security Analyzer
+    apt: debsecan
+    _type: cli
   defaultbrowser:
     _bin: null
     _desc: This role performs the necessary logic to set the default browser on nearly any platform.