Merge pull request #43 from megabyte-labs/18_netdata_alerts

Set of Netdata Alert Definitions

home/.chezmoidata.yaml

@@ -744,6 +744,7 @@ softwareGroups:
   Security: &Security
     - bitwarden-cli
     - boringtun
+    - debsecan
     - envchain
     - envconsul
     - pony

home/.chezmoiexternal.toml.tmpl

@@ -131,6 +131,16 @@
     pull.args = ["--ff-only"]
 {{- end }}
 
+{{- if (lookPath "apt") }}
+### Netdata Debsecan
+[".local/share/netdata-debsecan"]
+    type = "git-repo"
+    url = "https://gitlab.com/nodiscc/netdata-debsecan.git"
+    refreshPeriod = "{{ $refreshPeriod }}"
+    clone.args = ["--depth", "1"]
+    pull.args = ["--ff-only"]
+{{- end }}
+
 ### Git Template
 # [".config/git/template/_/husky.sh"]
 #     type = "file"

home/.chezmoiscripts/universal/run_onchange_after_57-netdata.sh.tmpl

@@ -6,6 +6,9 @@
 #     This script connects Netdata with Netdata Cloud if Netdata is installed, the `NETDATA_TOKEN` is provided, and the
 #     `NETDATA_ROOM` is defined. This allows you to graphically browse through system metrics on all your connected devices
 #     from a single free web application.
+#
+#     This script installs additional alerts and enables notifications if Netdata is installed. Email notifications are configured
+#     using the provided primary email address. If the OS is Debian based, Netdata shows the number of CVEs in currently installed packages.
 
 {{ includeTemplate "universal/profile" }}
 {{ includeTemplate "universal/logg" }}
@@ -34,6 +37,68 @@ if command -v netdata-claim.sh > /dev/null; then
             logg info 'The `/sys/kernel/mm/ksm` directory does not exist so Netdata kernel optimizations are not being applied'
         fi
     fi
+
+    ### Install additional alerts and enable notifications
+    if command -v netdata > /dev/null; then
+        # Copy the additional alert definitions
+        if [ -d /usr/local/etc/netdata ]; then
+            NETDATA_ETC='/usr/local/etc/netdata/'
+        elif [ -d /etc/netdata ]; then
+            NETDATA_EDC='/etc/netdata'
+        else
+            logg error 'No etc location found for netdata' && exit 1
+        fi
+        logg info "Copying ${XDG_CONFIG_HOME:-$HOME/.config}/netdata/health.d/ to $NETDATA_ETC"
+        sudo cp -rf "Copying ${XDG_CONFIG_HOME:-$HOME/.config}/netdata/health.d/" "$NETDATA_ETC"
+
+        # Backup current health alarm configuration and apply new one
+        if [ -d /usr/local/lib/netdata ]; then
+            NETDATA_LIB='/usr/local/lib/netdata'
+        elif [ -d /usr/lib/netdata ]; then
+            NETDATA_LIB='/usr/lib/netdata'
+        else
+            logg error 'No lib location found for netdata' && exit 1
+        fi
+        logg info "Copying $${XDG_CONFIG_HOME:-$HOME/.config}/netdata/health_alarm_notify.conf to $NETDATA_LIB/conf.d/health_alarm_notify.conf"
+        sudo cp -f "${XDG_CONFIG_HOME:-$HOME/.config}/netdata/health_alarm_notify.conf" "$NETDATA_LIB/conf.d/health_alarm_notify.conf"
+    else
+        logg warn '`netdata` is not available in the PATH or is not installed'
+    fi
+
+    ### Ensure the apt command is available before running `debsecan` logic
+    if command -v apt > /dev/null; then
+        ### Configure Netdata to gather information about CVEs in the installed packages
+        if command -v debsecan > /dev/null; then
+            DEBSECAN_GIT="${XDG_DATA_HOME:-$HOME/.local/share}/netdata-debsecan"
+
+            # Installing the script to generate report on CVEs in installed packages
+            logg info 'Installing script to generate on CVEs in installed packages'
+            sudo cp -f "$DEBSECAN_GIT/usr_local_bin_debsecan-by-type" /usr/local/bin/debsecan-by-type
+
+            # Generate initial debsecan reports in /var/log/debsecan/
+            logg info 'Generating initial debsecan reports in /var/log/debsecan/'
+            debsecan-by-type
+
+            # Configure dpkg to refresh the file after each run
+            logg info 'Configuring dpkg to refresh the file after each run'
+            sudo cp -f "$DEBSECAN_GIT/etc_apt_apt.conf.d_99debsecan"  /etc/apt/apt.conf.d/99-debsecan
+
+            # Add a cron job to refresh the file every hour
+            logg info 'Adding a cron job to refresh the file every hour'
+            sudo cp -f "$DEBSECAN_GIT/etc_cron.d_debsecan" /etc/cron.d/debsecan
+
+            # Install the module/configuration file
+            logg info 'Installing the module and configuration file
+            sudo "$DEBSECAN_GIT/debsecan.chart.py" /usr/libexec/netdata/python.d/debsecan.chart.py
+            sudo "$DEBSECAN_GIT/debsecan.conf" /etc/netdata/python.d/debsecan.conf
+
+            # Restart Netdata service
+            logg info 'Restarting netdata service'
+            sudo systemctl restart netdata
+        else
+            logg warn '`debsecan` is not available in the PATH or is not installed'
+        fi
+    fi
 else
     logg warn '`netdata-claim.sh` is not available in the PATH'
 fi

home/dot_config/netdata/health.d/apps.conf

@@ -0,0 +1,26 @@
+alarm: apps_cpu_usage
+on: apps.cpu
+lookup: average -1m percentage *
+units: %
+every: 5m
+warn: $this > 50
+crit: $this > 70
+info: CPU utilization of applications
+
+alarm: apps_ram_usage
+on: apps.mem
+lookup: average -1m percentage *
+units: %
+every: 5m
+warn: $this > 65
+crit: $this > 80
+info: RAM utilization of applications
+
+alarm: apps_swap_usage
+on: apps.swap
+lookup: average -1m percentage *
+units: %
+every: 5m
+warn: $this > 50
+crit: $this > 70
+info: Swap utilization of applications

home/dot_config/netdata/health.d/cpu.conf

@@ -0,0 +1,32 @@
+alarm: cpu_usage
+on: system.cpu
+lookup: average -1m percentage foreach user,system
+units: %
+every: 5m
+warn: $this > 50
+crit: $this > 70
+info: CPU utilization of user or the system
+
+template: ml_5min_cpu_dims
+on: system.cpu
+os: linux
+hosts: *
+lookup: average -5m anomaly-bit foreach *
+calc: $this
+units: %
+every: 30s
+warn: $this > (($status >= $WARNING)  ? (5) : (20))
+crit: $this > (($status == $CRITICAL) ? (20) : (100))
+info: Rolling 5 minute anomaly rate for each CPU dimension
+
+template: ml_5min_cpu_chart
+on: system.cpu
+os: linux
+hosts: *
+lookup: average -5m anomaly-bit of *
+calc: $this
+units: %
+every: 30s
+warn: $this > (($status >= $WARNING)  ? (5) : (20))
+crit: $this > (($status == $CRITICAL) ? (20) : (100))
+info: Rolling 5 minute anomaly rate for CPU chart

home/dot_config/netdata/health.d/disks.conf

@@ -0,0 +1,7 @@
+template: disk_full_percent
+on: disk.space
+calc: $used * 100 / ($avail + $used)
+every: 60m
+warn: $this > 70
+crit: $this > 85
+info: System disk usage

home/dot_config/netdata/health.d/network.conf

@@ -0,0 +1,6 @@
+template: 30min_packet_drops
+on: net.drops
+lookup: sum -30m unaligned absolute
+every: 10s
+crit: $this > 0
+info: Dropped network packets

home/dot_config/netdata/health.d/ram.conf

@@ -0,0 +1,8 @@
+alarm: ram_usage
+on: system.ram
+lookup: average -1m percentage foreach user,system
+units: %
+every: 5m
+warn: $this > 50
+crit: $this > 90
+info: RAM utilization of users or the system

software.yml

@@ -1759,6 +1759,15 @@ softwarePackages:
     _name: Debloat Windows
     ansible: professormanhattan.debloat
     _type: cli
+  debsecan:
+    _bin: debsecan-by-type
+    _desc: `debsecan` analyzes the list of installed packages on the current host and reports vulnerabilities found on the system.
+    _docs: null
+    _github: null
+    _gitlab: https://gitlab.com/fweimer/debsecan
+    _home: https://wiki.debian.org/DebianSecurity/debsecan
+    _name: Debian Security Analyzer
+    apt: debsecan
   defaultbrowser:
     _bin: null
     _desc: This role performs the necessary logic to set the default browser on nearly any platform.
@@ -2710,7 +2719,7 @@ softwarePackages:
     cask: xcodes
   nitroshare:
     _bin: nitroshare
-    _desc: Network file transfer application for Windows, OS X, & Linux 
+    _desc: Network file transfer application for Windows, OS X, & Linux
     _docs: https://nitroshare.net/
     _github: https://github.com/nitroshare/nitroshare-desktop
     _home: https://nitroshare.net/