Manually reporting on the system

Here is a non exhaustive list of useful command to get the global status of a system.

lsb_release -a
uname -a
whoami
who
w
df -h
uptime
free -h
ps -eo user,comm,pid,ppid,%mem --sort -%mem
journalctl -x
systemctl list-units
systemctl list-timers
systemctl list-sockets --all
systemctl list-unit-files
hcitool dev
cat /etc/hosts
nmcli d
ip a
ip l
ip route
iptables -L
ss -tlpn
avahi-browse -arlt
lsof -iTCP -sTCP:LISTEN
ss -tu
lsof -iTCP -sTCP:ESTABLISHED
systemctl status fail2ban
fail2ban-client status ssh
fail2ban-client status ssh |grep "Currently banned" | cut -d ":" -f 2
lsmod

Automatically reporting on the system

Use a systemd service.

  • system-status.service
[Unit]
Description=System status

[Service]
Type=simple
Environment="NS_PASSWORD=FOO"
ExecStart=/home/user/secu/notify_stats.py

[Install]
WantedBy=multi-user.target
  • system-status.timer
[Unit]
Description=Daily send system status

[Timer]
OnCalendar=*-*-* 18:00:00
Persistent=true

[Install]
WantedBy=timers.target
  • the /home/user/secu/notify_stats.py script
#!/usr/bin/env python3

# sends server status by email

import os
import re
import subprocess
import smtplib
import datetime
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from subprocess import PIPE, run


sender = "reporter@example.com"
recipients = ["user@example.com"]
smtp_config = "stmp.example.com:587"

if __name__ == "__main__":

    cmds = {
            # Add or remove commands
            "Users": "w",
            "Logins": "last | head -10",
            "Memory": "free -h",
            "Disk": "df -h",
            "Fail2ban status": "fail2ban-client status",
            "Fail2ban status SSH": "fail2ban-client status sshd",
            "Fail2ban status VPN": "fail2ban-client status openvpn",
            "Fail2ban status Nginx 403": "fail2ban-client status nginx-403",
            "Fail2ban status Nginx 444": "fail2ban-client status nginx-444",
            "Fail2ban status Nginx nowordpress": "fail2ban-client status nginx-nowordpress",
            "Fail2ban status Nginx x00": "fail2ban-client status nginx-x00",
            "Fail2ban status Nginx noproxy": "fail2ban-client status nginx-noproxy",
            "Fail2ban status Nginx noscript": "fail2ban-client status nginx-noscript",
            "Connections": "ss -tupn",
            "Containers connections": "for i in $(docker ps -a --format \"table {{.ID}}\t{{.Names}}\t{{.Ports}}\t{{.Status}}\" | grep -iv \"Exited\" | awk '{print $2}'|grep -v ID);do echo \"__ $i __\";nsenter -t $(docker inspect -f '{{.State.Pid}}' $i) -n ss state established -tu | tr -s \" \" ;done",
            "Services": "systemctl list-units | grep '\.service' | grep running | sort",
            "Services failed": "systemctl list-units --failed",
    }

    debug = {
            "Listening": "ss -tulpn",
            "Processes": "ps -eo user,comm,pid,ppid,%mem --sort -%mem",
    }

    res = {}
    date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S%z")
    for name, cmd in cmds.items():
        output = subprocess.getoutput(cmd)
        res[name] = output

    debug_data = []
    for name, cmd in debug.items():
        output = subprocess.getoutput(cmd)
        debug_data.append("======================================")
        debug_data.append(output)
    debug_data = "\n".join(debug_data)

    lines = []
    lines.append("*** Status of system ***")
    lines.append(f"date: {date}\n")
    for name, result in res.items():
        lines.append(f"============================\n{name}:\n{result}")
    message = "\n\n\n".join(lines)

    password = os.environ["NS_PASSWORD"]
    email_subject_line = f"[System] status"
    for recipient in recipients:
        msg = MIMEMultipart()
        msg['From'] = sender
        msg['To'] = recipient
        msg['Subject'] = email_subject_line

        email_body = message
        msg.attach(MIMEText(email_body, 'plain'))

        email_content = msg.as_string()
        server = smtplib.SMTP(smtp_config)
        server.starttls()
        server.login(sender, password)
        server.sendmail(sender, recipient, email_content)
        server.quit()