Alertmanager, blackbox_exporter, Grafana, Loki, node_exporter, Prometheus 安装

Alvis · 发表于 2024-12-4 14:38:47

马上注册，免受广告困扰，轻松兑换eSIM！

您需要登录才可以下载或查看，没有账号？注册

×

以下含 bash 的内容在我 CodeSpace 里睡了很久，遂分享。部分需要自己更改

Grafana install

看官网安装教程即可，很简单就不解释

Install Grafana | Grafana documentation

install_alertmanager.sh

注意更改 --web.external-url 参数

#!/bin/bash

# Get the latest version number of alertmanager
VER=$(wget -qO- https://api.github.com/repos/prometheus/alertmanager/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')

# Determine the system architecture
ARCH=$(uname -m)
TYPE=""

# Match the architecture to the corresponding alertmanager binary
if [ "$ARCH" == "x86_64" ]; then
  TYPE="amd64"
elif [ "$ARCH" == "armv5l" ]; then
  TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
  TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
  TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
  TYPE="arm64"
fi

# Stop alertmanager service
systemctl stop alertmanager

# Download the alertmanager binary for the detected architecture
wget -P /tmp https://github.com/prometheus/alertmanager/releases/download/v${VER}/alertmanager-${VER}.linux-${TYPE}.tar.gz

# Unpack the downloaded binary
tar -zxvf /tmp/alertmanager-${VER}.linux-${TYPE}.tar.gz -C /tmp

# Create a user for alertmanager without a home directory and with /bin/false as shell
useradd -rs /bin/false alertmanager

# Create configuration and data directories for alertmanager
mkdir -p /etc/alertmanager/templates
mkdir /etc/alertmanager
mkdir /var/lib/alertmanager

# Set ownership of the directories to the alertmanager user
chown alertmanager:alertmanager /etc/alertmanager
chown alertmanager:alertmanager /var/lib/alertmanager
chown -R alertmanager:alertmanager /etc/alertmanager/templates

# Check if the alertmanager.yml configuration file already exists
if [ ! -f /etc/alertmanager/alertmanager.yml ]; then
    # Copy the new alertmanager configuration file to the /etc/alertmanager directory
    cp /tmp/alertmanager-${VER}.linux-${TYPE}/alertmanager.yml /etc/alertmanager/alertmanager.yml
else
    echo "The configuration file /etc/alertmanager/alertmanager.yml already exists and will not be overwritten."
fi

# Copy the binaries to their respective locations
cp /tmp/alertmanager-${VER}.linux-${TYPE}/alertmanager /usr/local/bin/
cp /tmp/alertmanager-${VER}.linux-${TYPE}/amtool /usr/local/bin/

# Remove old files and directories
rm /tmp/alertmanager-${VER}.linux-${TYPE}.tar.gz && rm -rf /tmp/alertmanager-${VER}.linux-${TYPE}

# Set ownership of the binaries to the alertmanager user
chown alertmanager:alertmanager /usr/local/bin/alertmanager
chown alertmanager:alertmanager /usr/local/bin/amtool
chown alertmanager:alertmanager /etc/alertmanager/alertmanager.yml

# Create a systemd service file for alertmanager
cat > /etc/systemd/system/alertmanager.service << EOF
[Unit]
Description=AlertManager Service
Documentation=https://github.com/prometheus/alertmanager
Wants=network-online.target
After=network-online.target

[Service]
User=alertmanager
Group=alertmanager
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/alertmanager \\
    --config.file=/etc/alertmanager/alertmanager.yml \\
    --storage.path=/var/lib/alertmanager/ \\
    --web.external-url=https://status.microcharon.dev/alertmanager/ \\
    --web.listen-address=localhost:9093 \\
    --web.route-prefix=/

[Install]
WantedBy=multi-user.target
EOF

# Reload the systemd manager configuration
systemctl daemon-reload

# Enable the alertmanager service to start on boot
systemctl enable alertmanager

# Start the alertmanager service
systemctl start alertmanager

install_blackbox_exporter.sh

#!/bin/bash

# Get the latest version number of blackbox_exporter
VER=$(wget -qO- https://api.github.com/repos/prometheus/blackbox_exporter/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')

# Determine the system architecture
ARCH=$(uname -m)
TYPE=""

# Match the architecture to the corresponding blackbox_exporter binary
if [ "$ARCH" == "x86_64" ]; then
  TYPE="amd64"
elif [ "$ARCH" == "armv5l" ]; then
  TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
  TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
  TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
  TYPE="arm64"
fi

# Stop blackbox_exporter service
systemctl stop blackbox_exporter

# Download the blackbox_exporter binary for the detected architecture
wget -P /tmp https://github.com/prometheus/blackbox_exporter/releases/download/v${VER}/blackbox_exporter-${VER}.linux-${TYPE}.tar.gz

# Unpack the downloaded binary
tar -zxvf /tmp/blackbox_exporter-${VER}.linux-${TYPE}.tar.gz -C /tmp

# Create configuration and data directories for blackbox_exporter
mkdir /etc/blackbox_exporter

# Check if the blackbox.yml configuration file already exists
if [ ! -f /etc/blackbox_exporter/blackbox.yml ]; then
    # Copy the new blackbox_exporter configuration file to the /etc/blackbox_exporter directory
    cp /tmp/blackbox_exporter-${VER}.linux-${TYPE}/blackbox.yml /etc/blackbox_exporter/blackbox.yml
else
    echo "The configuration file /etc/blackbox_exporter/blackbox.yml already exists and will not be overwritten."
fi

# Copy the binaries to their respective locations
cp /tmp/blackbox_exporter-${VER}.linux-${TYPE}/blackbox_exporter /usr/local/bin/

# Remove old files and directories
rm /tmp/blackbox_exporter-${VER}.linux-${TYPE}.tar.gz && rm -rf /tmp/blackbox_exporter-${VER}.linux-${TYPE}

# Create a systemd service file for blackbox_exporter
cat > /etc/systemd/system/blackbox_exporter.service << EOF
[Unit]
Description=Blackbox Exporter Service
Documentation=https://github.com/prometheus/blackbox_exporter
Wants=network-online.target
After=network-online.target

[Service]
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/blackbox_exporter \\
    --config.file=/etc/blackbox_exporter/blackbox.yml \\
    --web.listen-address=:9115

[Install]
WantedBy=multi-user.target
EOF

# Reload the systemd manager configuration
systemctl daemon-reload

# Enable the blackbox_exporter service to start on boot
systemctl enable blackbox_exporter

# Start the blackbox_exporter service
systemctl start blackbox_exporter

install_loki.sh

# Get the latest version number of Loki
VER=$(wget -qO- https://api.github.com/repos/grafana/loki/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')

# Determine the system architecture
ARCH=$(uname -m)
TYPE=""

# Match the architecture to the corresponding Loki and Promtail binary files
if [ "$ARCH" == "x86_64" ]; then
  TYPE="amd64"
elif [ "$ARCH" == "armv7l" ]; then
  TYPE="arm"
elif [ "$ARCH" == "aarch64" ]; then
  TYPE="arm64"
fi

# Exit the script if the architecture is not supported
if [ -z "$TYPE" ]; then
  echo "Unsupported architecture: $ARCH"
  exit 1
fi

# Stop Loki and Promtail service
systemctl stop loki
systemctl stop promtail

# Create system users for Loki and Promtail
useradd -rs /bin/false loki
useradd -rs /bin/false promtail

# Download the Loki and Promtail binary files to the /tmp directory
wget https://github.com/grafana/loki/releases/download/v$VER/loki-linux-$TYPE.zip -O /tmp/loki.zip
wget https://github.com/grafana/loki/releases/download/v$VER/promtail-linux-$TYPE.zip -O /tmp/promtail.zip

# Unzip Loki and Promtail to /usr/local/bin/
unzip /tmp/loki.zip -d /usr/local/bin/
unzip /tmp/promtail.zip -d /usr/local/bin/
mv /usr/local/bin/loki-linux-$TYPE /usr/local/bin/loki
mv /usr/local/bin/promtail-linux-$TYPE /usr/local/bin/promtail

# Remove the downloaded zip files
rm /tmp/loki.zip
rm /tmp/promtail.zip

# Grant execution permissions
chmod +x /usr/local/bin/loki
chmod +x /usr/local/bin/promtail

# Download the configuration files
wget https://raw.githubusercontent.com/grafana/loki/main/cmd/loki/loki-local-config.yaml -O /etc/loki/loki.yaml
wget https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml -O /etc/promtail/promtail.yaml

# Create directories for configuration files
mkdir -p /etc/loki
mkdir -p /etc/promtail

# Set up the Loki service
cat > /etc/systemd/system/loki.service << EOF
[Unit]
Description=Loki Service
Documentation=https://github.com/grafana/loki
After=network.target

[Service]
User=loki
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/loki -config.file=/etc/loki/loki.yaml

[Install]
WantedBy=multi-user.target
EOF

# Set up the Promtail service
cat > /etc/systemd/system/promtail.service << EOF
[Unit]
Description=Promtail Service
Documentation=https://github.com/grafana/loki
After=network.target

[Service]
User=promtail
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/promtail -config.file=/etc/promtail/promtail.yaml

[Install]
WantedBy=multi-user.target
EOF

# since it uses Promtail to read system log files, 
# the promtail user won't yet have permissions to read them.
usermod -a -G adm promtail

# Start and enable the services
systemctl daemon-reload

systemctl enable loki
systemctl start loki

systemctl enable promtail
systemctl start promtail

install_node_exporter.sh

#!/bin/bash

VER=$(curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')

ARCH=$(uname -m)
TYPE=""

if [ "$ARCH" == "x86_64" ]; then
  TYPE="amd64"
elif [ "$ARCH" == "arm5l" ]; then
  TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
  TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
  TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
  TYPE="arm64"
fi

# Stop node_exporter service
systemctl stop node_exporter

# Wait for the service to stop
while [ "$(systemctl is-active node_exporter)" == "active" ]; do
  sleep 1
done

# Download the newest node_exporter
wget https://github.com/prometheus/node_exporter/releases/download/v${VER}/node_exporter-${VER}.linux-${TYPE}.tar.gz

# Copy the node_exporter
tar -zxvf node_exporter*.tar.gz && cp ./node_exporter-${VER}.linux-${TYPE}/node_exporter /usr/local/bin

# Remove old files and directories
rm node_exporter*.tar.gz node_exporter*/* && rmdir node_exporter-${VER}.linux-${TYPE}

# Create systemd service file
cat > /etc/systemd/system/node_exporter.service << "EOF"
[Unit]
Description=Node Exporter Service
Documentation=https://github.com/prometheus/node_exporter

[Service]
ExecStart=/usr/local/bin/node_exporter --web.listen-address=:9100
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

# Reload units
systemctl daemon-reload

# Enable node_exporter service to start on boot
systemctl enable node_exporter

# Restart node_exporter service
systemctl start node_exporter

install_prometheus.sh

注意更改 --web.external-url 参数

#!/bin/bash

# Get the latest version number
VER=$(wget -qO- https://api.github.com/repos/prometheus/prometheus/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')

# Determine the system architecture
ARCH=$(uname -m)
TYPE=""

if [ "$ARCH" == "x86_64" ]; then
  TYPE="amd64"
elif [ "$ARCH" == "arm5l" ]; then
  TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
  TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
  TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
  TYPE="arm64"
fi

# Stop prometheus service
systemctl stop prometheus

# Download the corresponding architecture's Prometheus to the /tmp directory
wget -P /tmp https://github.com/prometheus/prometheus/releases/download/v${VER}/prometheus-${VER}.linux-${TYPE}.tar.gz

# Unpack the file
tar -zxvf /tmp/prometheus-${VER}.linux-${TYPE}.tar.gz -C /tmp

# Create user and configuration directories
useradd -rs /bin/false prometheus
mkdir /etc/prometheus
mkdir /var/lib/prometheus
chown prometheus:prometheus /etc/prometheus
chown prometheus:prometheus /var/lib/prometheus

# Check if the prometheus.yml configuration file already exists
if [ ! -f /etc/prometheus/prometheus.yml ]; then
    # Copy the new prometheus configuration file to the /etc/prometheus directory
    cp /tmp/prometheus-${VER}.linux-${TYPE}/prometheus.yml /etc/prometheus/prometheus.yml
else
    echo "The configuration file /etc/prometheus/prometheus.yml already exists and will not be overwritten."
fi

chown prometheus:prometheus /etc/prometheus/prometheus.yml

# Copy binaries and configuration files
cp /tmp/prometheus-${VER}.linux-${TYPE}/prometheus /usr/local/bin/
cp /tmp/prometheus-${VER}.linux-${TYPE}/promtool /usr/local/bin/
chown prometheus:prometheus /usr/local/bin/prometheus
chown prometheus:prometheus /usr/local/bin/promtool

cp -r /tmp/prometheus-${VER}.linux-${TYPE}/consoles /etc/prometheus
cp -r /tmp/prometheus-${VER}.linux-${TYPE}/console_libraries /etc/prometheus
chown -R prometheus:prometheus /etc/prometheus/consoles
chown -R prometheus:prometheus /etc/prometheus/console_libraries

# Remove old files and directories
rm /tmp/prometheus-${VER}.linux-${TYPE}.tar.gz && rm -rf /tmp/prometheus-${VER}.linux-${TYPE}

# Create systemd service file
cat > /etc/systemd/system/prometheus.service << EOF 
[Unit]
Description=Prometheus Service
Documentation=https://github.com/prometheus/prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/prometheus \\
    --config.file /etc/prometheus/prometheus.yml \\
    --storage.tsdb.path /var/lib/prometheus/ \\
    --web.console.templates=/etc/prometheus/consoles \\
    --web.console.libraries=/etc/prometheus/console_libraries \\
    --web.enable-lifecycle \\
    --web.enable-admin-api \\
    --web.external-url=https://status.microcharon.dev/prometheus/ \\
    --web.listen-address=localhost:9090 \\
    --web.route-prefix=/

[Install]
WantedBy=multi-user.target
EOF

# Reload systemd manager configuration
systemctl daemon-reload

# Enable Prometheus service to start on boot
systemctl enable prometheus

# Start Prometheus service
systemctl start prometheus

Vhost Configuration

RSS & Atom Reader - Dashboards - Grafana

以下是自己 Grafana 实例的 NGINX Vhost 配置，仅供参考

upstream grafana {
    server localhost:3000;
}

upstream prometheus {
    server localhost:9090;
}

upstream alertmanager {
    server localhost:9093;
}

server
    {
        listen 80;
        listen [::]:80;
        server_name status.microcharon.dev ;
        index index.html index.htm index.php default.html default.htm default.php;
        root  /home/wwwroot/status.microcharon.dev;

        #include rewrite/none.conf;
        #error_page   404   /404.html;

        # Deny access to PHP files in specific directory
        #location ~ /(wp-content|uploads|wp-includes|images)/.*\.php$ { deny all; }

        include enable-php.conf;

        location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$
        {
            expires      30d;
        }

        location ~ .*\.(js|css)?$
        {
            expires      12h;
        }

        location ~ /.well-known {
            allow all;
        }

        location ~ /\.
        {
            deny all;
        }

#        location / {
#            return 301 https://$host$request_uri;
#        }

        access_log  /home/wwwlogs/status.microcharon.dev.log;
    }

server
    {
        #listen 443 ssl http2;
        #listen [::]:443 ssl http2;
        listen 443 ssl;
        listen [::]:443 ssl;
        server_name status.microcharon.dev ;
        index index.html index.htm index.php default.html default.htm default.php;
        root  /home/wwwroot/status.microcharon.dev;

        ssl_certificate /usr/local/nginx/conf/ssl/status.microcharon.dev/fullchain.cer;
        ssl_certificate_key /usr/local/nginx/conf/ssl/status.microcharon.dev/status.microcharon.dev.key;
        ssl_session_timeout 5m;
        ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
        ssl_prefer_server_ciphers on;
        ssl_ciphers "TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:TLS13-AES-128-GCM-SHA256:TLS13-AES-128-CCM-8-SHA256:TLS13-AES-128-CCM-SHA256:EECDH+CHACHA20:EECDH+CHACHA20-draft:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5";
        ssl_session_cache builtin:1000 shared:SSL:10m;
        # openssl dhparam -out /usr/local/nginx/conf/ssl/dhparam.pem 2048
        ssl_dhparam /usr/local/nginx/conf/ssl/dhparam.pem;

        include rewrite/none.conf;
        #error_page   404   /404.html;

        # Deny access to PHP files in specific directory
        #location ~ /(wp-content|uploads|wp-includes|images)/.*\.php$ { deny all; }

        include enable-php.conf;

        location / {
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "Upgrade";
            proxy_redirect off;
            proxy_set_header Host $http_host;
            #proxy_set_header True-Client-IP $http_true_client_ip;
            proxy_set_header X-Real-IP $http_true_client_ip;
            #proxy_set_header X-Forwarded-For $remote_addr;
            #proxy_set_header X-Forwarded-For $http_true_client_ip;
            #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_pass http://grafana;
        }

        location /api/live {
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "Upgrade";
            proxy_set_header Host $http_host;
            proxy_pass http://grafana;
            rewrite ^/(.*)  /$1 break;
        }

        location /prometheus/ {
            proxy_pass http://prometheus/;
            auth_basic "Prometheus";
            auth_basic_user_file /home/wwwroot/status.microcharon.dev/.htpasswd;
        }

        location /alertmanager/ {
            proxy_pass http://alertmanager/;
            auth_basic "Alertmanager";
            auth_basic_user_file /home/wwwroot/status.microcharon.dev/.htpasswd;
        }

        access_log  /home/wwwlogs/status.microcharon.dev.log;
    }

Blackbox exporter - ICMP，仅供参考

目录结构

root@hetzner-fsn1-002:/etc/prometheus# tree
.
├── console_libraries
│   ├── menu.lib
│   └── prom.lib
├── consoles
│   ├── index.html.example
│   ├── node-cpu.html
│   ├── node-disk.html
│   ├── node.html
│   ├── node-overview.html
│   ├── prometheus.html
│   └── prometheus-overview.html
├── prometheus.yml
├── rules
│   ├── api_alert_rules.yml
│   ├── blackbox_exporter_alert_rules.yml
│   ├── node_exporter_alert_rules.yml
│   └── node_usage_record_rules.yml
└── services
    ├── your-service1.json
    ├── your-service2.json
    ├── your-service3.json
    ├── icmp_test1.json
    ├── icmp_test2.json

prometheus.yml 中 scrape_configs 配置

scrape_configs:

  - job_name: "icmp_test1"
    metrics_path: /probe
    scrape_interval: 1m
    params:
      module: [icmp]
    file_sd_configs:
    - files:
      - 'services/icmp_test1.json'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
     # - source_labels: [__param_target]
     #  target_label: instance
      - target_label: __address__
        replacement: test1.compute.microcharon.com:9115

  - job_name: "icmp_test2"
    metrics_path: /probe
    scrape_interval: 1m
    params:
      module: [icmp]
    file_sd_configs:
    - files:
      - 'services/icmp_test2.json'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
     # - source_labels: [__param_target]
     #  target_label: instance
      - target_label: __address__
        replacement: test2.compute.microcharon.com:9115

icmp_test1.json

[
  {
    "targets": [
      "223.5.5.5"
    ],
    "labels": {
      "instance": "tencent-dns"
    }
  },
  {
    "targets": [
      "219.151.141.1"
    ],
    "labels": {
      "instance": "chongqing-chinatelecom"
    }
  },
  {
    "targets": [
      "113.207.73.129"
    ],
    "labels": {
      "instance": "chongqing-chinaunicom"
    }
  },
  {
    "targets": [
      "211.139.55.65"
    ],
    "labels": {
      "instance": "chongqing-chinamobile"
    }
  },
  {
    "targets": [
      "203.80.96.10"
    ],
    "labels": {
      "instance": "hkbn-dns"
    }
  }
]