Alertmanager, blackbox_exporter, Grafana, Loki, node_exporter, Prometheus 安装
以下含 bash 的内容在我 CodeSpace 里睡了很久,遂分享。部分需要自己更改### Grafana install
看官网安装教程即可,很简单就不解释
(https://grafana.com/docs/grafana/latest/setup-grafana/installation/)
### install_alertmanager.sh
注意更改 `--web.external-url` 参数
```bash
#!/bin/bash
# Get the latest version number of alertmanager
VER=$(wget -qO- https://api.github.com/repos/prometheus/alertmanager/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
# Determine the system architecture
ARCH=$(uname -m)
TYPE=""
# Match the architecture to the corresponding alertmanager binary
if [ "$ARCH" == "x86_64" ]; then
TYPE="amd64"
elif [ "$ARCH" == "armv5l" ]; then
TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
TYPE="arm64"
fi
# Stop alertmanager service
systemctl stop alertmanager
# Download the alertmanager binary for the detected architecture
wget -P /tmp https://github.com/prometheus/alertmanager/releases/download/v${VER}/alertmanager-${VER}.linux-${TYPE}.tar.gz
# Unpack the downloaded binary
tar -zxvf /tmp/alertmanager-${VER}.linux-${TYPE}.tar.gz -C /tmp
# Create a user for alertmanager without a home directory and with /bin/false as shell
useradd -rs /bin/false alertmanager
# Create configuration and data directories for alertmanager
mkdir -p /etc/alertmanager/templates
mkdir /etc/alertmanager
mkdir /var/lib/alertmanager
# Set ownership of the directories to the alertmanager user
chown alertmanager:alertmanager /etc/alertmanager
chown alertmanager:alertmanager /var/lib/alertmanager
chown -R alertmanager:alertmanager /etc/alertmanager/templates
# Check if the alertmanager.yml configuration file already exists
if [ ! -f /etc/alertmanager/alertmanager.yml ]; then
# Copy the new alertmanager configuration file to the /etc/alertmanager directory
cp /tmp/alertmanager-${VER}.linux-${TYPE}/alertmanager.yml /etc/alertmanager/alertmanager.yml
else
echo "The configuration file /etc/alertmanager/alertmanager.yml already exists and will not be overwritten."
fi
# Copy the binaries to their respective locations
cp /tmp/alertmanager-${VER}.linux-${TYPE}/alertmanager /usr/local/bin/
cp /tmp/alertmanager-${VER}.linux-${TYPE}/amtool /usr/local/bin/
# Remove old files and directories
rm /tmp/alertmanager-${VER}.linux-${TYPE}.tar.gz && rm -rf /tmp/alertmanager-${VER}.linux-${TYPE}
# Set ownership of the binaries to the alertmanager user
chown alertmanager:alertmanager /usr/local/bin/alertmanager
chown alertmanager:alertmanager /usr/local/bin/amtool
chown alertmanager:alertmanager /etc/alertmanager/alertmanager.yml
# Create a systemd service file for alertmanager
cat > /etc/systemd/system/alertmanager.service << EOF
Description=AlertManager Service
Documentation=https://github.com/prometheus/alertmanager
Wants=network-online.target
After=network-online.target
User=alertmanager
Group=alertmanager
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/alertmanager \\
--config.file=/etc/alertmanager/alertmanager.yml \\
--storage.path=/var/lib/alertmanager/ \\
--web.external-url=https://status.microcharon.dev/alertmanager/ \\
--web.listen-address=localhost:9093 \\
--web.route-prefix=/
WantedBy=multi-user.target
EOF
# Reload the systemd manager configuration
systemctl daemon-reload
# Enable the alertmanager service to start on boot
systemctl enable alertmanager
# Start the alertmanager service
systemctl start alertmanager
```
### install_blackbox_exporter.sh
```bash
#!/bin/bash
# Get the latest version number of blackbox_exporter
VER=$(wget -qO- https://api.github.com/repos/prometheus/blackbox_exporter/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
# Determine the system architecture
ARCH=$(uname -m)
TYPE=""
# Match the architecture to the corresponding blackbox_exporter binary
if [ "$ARCH" == "x86_64" ]; then
TYPE="amd64"
elif [ "$ARCH" == "armv5l" ]; then
TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
TYPE="arm64"
fi
# Stop blackbox_exporter service
systemctl stop blackbox_exporter
# Download the blackbox_exporter binary for the detected architecture
wget -P /tmp https://github.com/prometheus/blackbox_exporter/releases/download/v${VER}/blackbox_exporter-${VER}.linux-${TYPE}.tar.gz
# Unpack the downloaded binary
tar -zxvf /tmp/blackbox_exporter-${VER}.linux-${TYPE}.tar.gz -C /tmp
# Create configuration and data directories for blackbox_exporter
mkdir /etc/blackbox_exporter
# Check if the blackbox.yml configuration file already exists
if [ ! -f /etc/blackbox_exporter/blackbox.yml ]; then
# Copy the new blackbox_exporter configuration file to the /etc/blackbox_exporter directory
cp /tmp/blackbox_exporter-${VER}.linux-${TYPE}/blackbox.yml /etc/blackbox_exporter/blackbox.yml
else
echo "The configuration file /etc/blackbox_exporter/blackbox.yml already exists and will not be overwritten."
fi
# Copy the binaries to their respective locations
cp /tmp/blackbox_exporter-${VER}.linux-${TYPE}/blackbox_exporter /usr/local/bin/
# Remove old files and directories
rm /tmp/blackbox_exporter-${VER}.linux-${TYPE}.tar.gz && rm -rf /tmp/blackbox_exporter-${VER}.linux-${TYPE}
# Create a systemd service file for blackbox_exporter
cat > /etc/systemd/system/blackbox_exporter.service << EOF
Description=Blackbox Exporter Service
Documentation=https://github.com/prometheus/blackbox_exporter
Wants=network-online.target
After=network-online.target
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/blackbox_exporter \\
--config.file=/etc/blackbox_exporter/blackbox.yml \\
--web.listen-address=:9115
WantedBy=multi-user.target
EOF
# Reload the systemd manager configuration
systemctl daemon-reload
# Enable the blackbox_exporter service to start on boot
systemctl enable blackbox_exporter
# Start the blackbox_exporter service
systemctl start blackbox_exporter
```
### install_loki.sh
```bash
# Get the latest version number of Loki
VER=$(wget -qO- https://api.github.com/repos/grafana/loki/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
# Determine the system architecture
ARCH=$(uname -m)
TYPE=""
# Match the architecture to the corresponding Loki and Promtail binary files
if [ "$ARCH" == "x86_64" ]; then
TYPE="amd64"
elif [ "$ARCH" == "armv7l" ]; then
TYPE="arm"
elif [ "$ARCH" == "aarch64" ]; then
TYPE="arm64"
fi
# Exit the script if the architecture is not supported
if [ -z "$TYPE" ]; then
echo "Unsupported architecture: $ARCH"
exit 1
fi
# Stop Loki and Promtail service
systemctl stop loki
systemctl stop promtail
# Create system users for Loki and Promtail
useradd -rs /bin/false loki
useradd -rs /bin/false promtail
# Download the Loki and Promtail binary files to the /tmp directory
wget https://github.com/grafana/loki/releases/download/v$VER/loki-linux-$TYPE.zip -O /tmp/loki.zip
wget https://github.com/grafana/loki/releases/download/v$VER/promtail-linux-$TYPE.zip -O /tmp/promtail.zip
# Unzip Loki and Promtail to /usr/local/bin/
unzip /tmp/loki.zip -d /usr/local/bin/
unzip /tmp/promtail.zip -d /usr/local/bin/
mv /usr/local/bin/loki-linux-$TYPE /usr/local/bin/loki
mv /usr/local/bin/promtail-linux-$TYPE /usr/local/bin/promtail
# Remove the downloaded zip files
rm /tmp/loki.zip
rm /tmp/promtail.zip
# Grant execution permissions
chmod +x /usr/local/bin/loki
chmod +x /usr/local/bin/promtail
# Download the configuration files
wget https://raw.githubusercontent.com/grafana/loki/main/cmd/loki/loki-local-config.yaml -O /etc/loki/loki.yaml
wget https://raw.githubusercontent.com/grafana/loki/main/clients/cmd/promtail/promtail-local-config.yaml -O /etc/promtail/promtail.yaml
# Create directories for configuration files
mkdir -p /etc/loki
mkdir -p /etc/promtail
# Set up the Loki service
cat > /etc/systemd/system/loki.service << EOF
Description=Loki Service
Documentation=https://github.com/grafana/loki
After=network.target
User=loki
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/loki -config.file=/etc/loki/loki.yaml
WantedBy=multi-user.target
EOF
# Set up the Promtail service
cat > /etc/systemd/system/promtail.service << EOF
Description=Promtail Service
Documentation=https://github.com/grafana/loki
After=network.target
User=promtail
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/promtail -config.file=/etc/promtail/promtail.yaml
WantedBy=multi-user.target
EOF
# since it uses Promtail to read system log files,
# the promtail user won't yet have permissions to read them.
usermod -a -G adm promtail
# Start and enable the services
systemctl daemon-reload
systemctl enable loki
systemctl start loki
systemctl enable promtail
systemctl start promtail
```
### install_node_exporter.sh
```bash
#!/bin/bash
VER=$(curl -s https://api.github.com/repos/prometheus/node_exporter/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
ARCH=$(uname -m)
TYPE=""
if [ "$ARCH" == "x86_64" ]; then
TYPE="amd64"
elif [ "$ARCH" == "arm5l" ]; then
TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
TYPE="arm64"
fi
# Stop node_exporter service
systemctl stop node_exporter
# Wait for the service to stop
while [ "$(systemctl is-active node_exporter)" == "active" ]; do
sleep 1
done
# Download the newest node_exporter
wget https://github.com/prometheus/node_exporter/releases/download/v${VER}/node_exporter-${VER}.linux-${TYPE}.tar.gz
# Copy the node_exporter
tar -zxvf node_exporter*.tar.gz && cp ./node_exporter-${VER}.linux-${TYPE}/node_exporter /usr/local/bin
# Remove old files and directories
rm node_exporter*.tar.gz node_exporter*/* && rmdir node_exporter-${VER}.linux-${TYPE}
# Create systemd service file
cat > /etc/systemd/system/node_exporter.service << "EOF"
Description=Node Exporter Service
Documentation=https://github.com/prometheus/node_exporter
ExecStart=/usr/local/bin/node_exporter --web.listen-address=:9100
Restart=on-failure
WantedBy=multi-user.target
EOF
# Reload units
systemctl daemon-reload
# Enable node_exporter service to start on boot
systemctl enable node_exporter
# Restart node_exporter service
systemctl start node_exporter
```
### install_prometheus.sh
注意更改 `--web.external-url` 参数
```bash
#!/bin/bash
# Get the latest version number
VER=$(wget -qO- https://api.github.com/repos/prometheus/prometheus/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
# Determine the system architecture
ARCH=$(uname -m)
TYPE=""
if [ "$ARCH" == "x86_64" ]; then
TYPE="amd64"
elif [ "$ARCH" == "arm5l" ]; then
TYPE="armv5"
elif [ "$ARCH" == "armv6l" ]; then
TYPE="armv6"
elif [ "$ARCH" == "armv7l" ]; then
TYPE="armv7"
elif [ "$ARCH" == "aarch64" ]; then
TYPE="arm64"
fi
# Stop prometheus service
systemctl stop prometheus
# Download the corresponding architecture's Prometheus to the /tmp directory
wget -P /tmp https://github.com/prometheus/prometheus/releases/download/v${VER}/prometheus-${VER}.linux-${TYPE}.tar.gz
# Unpack the file
tar -zxvf /tmp/prometheus-${VER}.linux-${TYPE}.tar.gz -C /tmp
# Create user and configuration directories
useradd -rs /bin/false prometheus
mkdir /etc/prometheus
mkdir /var/lib/prometheus
chown prometheus:prometheus /etc/prometheus
chown prometheus:prometheus /var/lib/prometheus
# Check if the prometheus.yml configuration file already exists
if [ ! -f /etc/prometheus/prometheus.yml ]; then
# Copy the new prometheus configuration file to the /etc/prometheus directory
cp /tmp/prometheus-${VER}.linux-${TYPE}/prometheus.yml /etc/prometheus/prometheus.yml
else
echo "The configuration file /etc/prometheus/prometheus.yml already exists and will not be overwritten."
fi
chown prometheus:prometheus /etc/prometheus/prometheus.yml
# Copy binaries and configuration files
cp /tmp/prometheus-${VER}.linux-${TYPE}/prometheus /usr/local/bin/
cp /tmp/prometheus-${VER}.linux-${TYPE}/promtool /usr/local/bin/
chown prometheus:prometheus /usr/local/bin/prometheus
chown prometheus:prometheus /usr/local/bin/promtool
cp -r /tmp/prometheus-${VER}.linux-${TYPE}/consoles /etc/prometheus
cp -r /tmp/prometheus-${VER}.linux-${TYPE}/console_libraries /etc/prometheus
chown -R prometheus:prometheus /etc/prometheus/consoles
chown -R prometheus:prometheus /etc/prometheus/console_libraries
# Remove old files and directories
rm /tmp/prometheus-${VER}.linux-${TYPE}.tar.gz && rm -rf /tmp/prometheus-${VER}.linux-${TYPE}
# Create systemd service file
cat > /etc/systemd/system/prometheus.service << EOF
Description=Prometheus Service
Documentation=https://github.com/prometheus/prometheus
Wants=network-online.target
After=network-online.target
User=prometheus
Group=prometheus
Type=simple
Restart=on-failure
ExecStart=/usr/local/bin/prometheus \\
--config.file /etc/prometheus/prometheus.yml \\
--storage.tsdb.path /var/lib/prometheus/ \\
--web.console.templates=/etc/prometheus/consoles \\
--web.console.libraries=/etc/prometheus/console_libraries \\
--web.enable-lifecycle \\
--web.enable-admin-api \\
--web.external-url=https://status.microcharon.dev/prometheus/ \\
--web.listen-address=localhost:9090 \\
--web.route-prefix=/
WantedBy=multi-user.target
EOF
# Reload systemd manager configuration
systemctl daemon-reload
# Enable Prometheus service to start on boot
systemctl enable prometheus
# Start Prometheus service
systemctl start prometheus
```
### Vhost Configuration
(https://status.microcharon.dev)
以下是自己 Grafana 实例的 NGINX Vhost 配置,仅供参考
```
upstream grafana {
server localhost:3000;
}
upstream prometheus {
server localhost:9090;
}
upstream alertmanager {
server localhost:9093;
}
server
{
listen 80;
listen [::]:80;
server_name status.microcharon.dev ;
index index.html index.htm index.php default.html default.htm default.php;
root/home/wwwroot/status.microcharon.dev;
#include rewrite/none.conf;
#error_page 404 /404.html;
# Deny access to PHP files in specific directory
#location ~ /(wp-content|uploads|wp-includes|images)/.*\.php$ { deny all; }
include enable-php.conf;
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$
{
expires 30d;
}
location ~ .*\.(js|css)?$
{
expires 12h;
}
location ~ /.well-known {
allow all;
}
location ~ /\.
{
deny all;
}
# location / {
# return 301 https://$host$request_uri;
# }
access_log/home/wwwlogs/status.microcharon.dev.log;
}
server
{
#listen 443 ssl http2;
#listen [::]:443 ssl http2;
listen 443 ssl;
listen [::]:443 ssl;
server_name status.microcharon.dev ;
index index.html index.htm index.php default.html default.htm default.php;
root/home/wwwroot/status.microcharon.dev;
ssl_certificate /usr/local/nginx/conf/ssl/status.microcharon.dev/fullchain.cer;
ssl_certificate_key /usr/local/nginx/conf/ssl/status.microcharon.dev/status.microcharon.dev.key;
ssl_session_timeout 5m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_ciphers "TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:TLS13-AES-128-GCM-SHA256:TLS13-AES-128-CCM-8-SHA256:TLS13-AES-128-CCM-SHA256:EECDH+CHACHA20:EECDH+CHACHA20-draft:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5";
ssl_session_cache builtin:1000 shared:SSL:10m;
# openssl dhparam -out /usr/local/nginx/conf/ssl/dhparam.pem 2048
ssl_dhparam /usr/local/nginx/conf/ssl/dhparam.pem;
include rewrite/none.conf;
#error_page 404 /404.html;
# Deny access to PHP files in specific directory
#location ~ /(wp-content|uploads|wp-includes|images)/.*\.php$ { deny all; }
include enable-php.conf;
location / {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_redirect off;
proxy_set_header Host $http_host;
#proxy_set_header True-Client-IP $http_true_client_ip;
proxy_set_header X-Real-IP $http_true_client_ip;
#proxy_set_header X-Forwarded-For $remote_addr;
#proxy_set_header X-Forwarded-For $http_true_client_ip;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://grafana;
}
location /api/live {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $http_host;
proxy_pass http://grafana;
rewrite ^/(.*)/$1 break;
}
location /prometheus/ {
proxy_pass http://prometheus/;
auth_basic "Prometheus";
auth_basic_user_file /home/wwwroot/status.microcharon.dev/.htpasswd;
}
location /alertmanager/ {
proxy_pass http://alertmanager/;
auth_basic "Alertmanager";
auth_basic_user_file /home/wwwroot/status.microcharon.dev/.htpasswd;
}
access_log/home/wwwlogs/status.microcharon.dev.log;
}
```
Blackbox exporter - ICMP,仅供参考
!(https://i.111666.best/image/aEDoqVvoBRyjOadm92YYcU.png)
目录结构
```bash
root@hetzner-fsn1-002:/etc/prometheus# tree
.
├── console_libraries
│ ├── menu.lib
│ └── prom.lib
├── consoles
│ ├── index.html.example
│ ├── node-cpu.html
│ ├── node-disk.html
│ ├── node.html
│ ├── node-overview.html
│ ├── prometheus.html
│ └── prometheus-overview.html
├── prometheus.yml
├── rules
│ ├── api_alert_rules.yml
│ ├── blackbox_exporter_alert_rules.yml
│ ├── node_exporter_alert_rules.yml
│ └── node_usage_record_rules.yml
└── services
├── your-service1.json
├── your-service2.json
├── your-service3.json
├── icmp_test1.json
├── icmp_test2.json
```
prometheus.yml 中 scrape_configs 配置
```yaml
scrape_configs:
- job_name: "icmp_test1"
metrics_path: /probe
scrape_interval: 1m
params:
module:
file_sd_configs:
- files:
- 'services/icmp_test1.json'
relabel_configs:
- source_labels:
target_label: __param_target
# - source_labels:
#target_label: instance
- target_label: __address__
replacement: test1.compute.microcharon.com:9115
- job_name: "icmp_test2"
metrics_path: /probe
scrape_interval: 1m
params:
module:
file_sd_configs:
- files:
- 'services/icmp_test2.json'
relabel_configs:
- source_labels:
target_label: __param_target
# - source_labels:
#target_label: instance
- target_label: __address__
replacement: test2.compute.microcharon.com:9115
```
icmp_test1.json
```json
[
{
"targets": [
"223.5.5.5"
],
"labels": {
"instance": "tencent-dns"
}
},
{
"targets": [
"219.151.141.1"
],
"labels": {
"instance": "chongqing-chinatelecom"
}
},
{
"targets": [
"113.207.73.129"
],
"labels": {
"instance": "chongqing-chinaunicom"
}
},
{
"targets": [
"211.139.55.65"
],
"labels": {
"instance": "chongqing-chinamobile"
}
},
{
"targets": [
"203.80.96.10"
],
"labels": {
"instance": "hkbn-dns"
}
}
]
```
页:
[1]