wget https://github.com/prometheus/prometheus/releases/download/v2.26.0-rc.0/prometheus-2.26.0-rc.0.linux-amd64.tar.gz
tar -xf prometheus-2.26.0-rc.0.linux-amd64.tar.gz
mv prometheus-2.26.0-rc.0.linux-amd64 prometheus

groupadd prometheus
useradd -g prometheus -s /sbin/nologin prometheus
mkdir /root/prometheus/data
chown -R prometheus:prometheus /root/prometheus

vim /root/prometheus/start.sh
#!/bin/bash
prometheus_dir=/root/prometheus
${prometheus_dir}/prometheus --config.file=${prometheus_dir}/prometheus.yml --storage.tsdb.path=${prometheus_dir}/data --storage.tsdb.retention.time=24h --web.enable-lifecycle --storage.tsdb.no-lockfile
# --config.file:指定配置文件路径
# --storage.tsdb.path:指定tsdb路径
# --storage.tsdb.retention.time:指定数据存储时间
# --web.enable-lifecycle:类似nginx的reload功能
# --storage.tsdb.no-lockfile:如果用k8s的deployment管理需加此项

# nohup英文全称no hang up（不挂起），用于在系统后台不挂断地运行命令，退出终端不会影响程序的运行。
nohup sh start.sh 2>&1 > prometheus.log

yum -y install screen
# 进入后台
screen
# 运行脚本
/root/prometheus/prometheus --config.file=/root/prometheus/prometheus.yml --storage.tsdb.path=/root/prometheus/data --storage.tsdb.retention.time=24h --web.enable-lifecycle --storage.tsdb.no-lockfile
# 输入CTRL + A + D撤回前台
# 查看后台运行的脚本
screen -ls
# 返回后台
screen -r 后台id
# 删除后台
screen -S 后台id -X quit

wget https://github.com/prometheus/node_exporter/releases/download/v1.1.2/node_exporter-1.1.2.linux-amd64.tar.gz
tar -xf node_exporter-1.1.2.linux-amd64.tar.gz
mv node_exporter-1.1.2.linux-amd64.tar.gz node_exporter

screen
./root/node_exporter/node_exporter

curl http://localhost:9100/metrics
...

node_cpu_seconds_total{mode='idle'}

increase(node_cpu_seconds_total{mode='idle'}[1m])

sum(increase(node_cpu_seconds_total{mode='idle'}[1m]))

sum(increase(node_cpu_seconds_total{mode='idle'}[1m])) by(instance)

(1 - ((sum(increase(node_cpu_seconds_total{mode='idle'}[1m])) by(instance)) / (sum(increase(node_cpu_seconds_total[1m])) by(instance)))) * 100

(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

rate(node_network_receive_bytes_total{device='ens33'}[1m])

count((1 - ((sum(increase(node_cpu_seconds_total{mode='idle'}[1m])) by(instance)) / (sum(increase(node_cpu_seconds_total[1m])) by(instance)))) * 100 > 80)

wget https://github.com/prometheus/pushgateway/releases/download/v1.4.0/pushgateway-1.4.0.linux-amd64.tar.gz
tar -xf pushgateway-1.4.0.linux-amd64.tar.gz
mv pushgateway-1.4.0.linux-amd64 pushgateway

screen
./root/pushgateway/pushgateway

- job_name: 'pushgateway'
  static_configs:
  - targets: ['localhost:9091']

#!/bin/bash

# 获取监控主机名
instance_name=`hostname -f | cut -d'.' -f1`

# 如果主机名为localhost，则退出
if [ $instance_name == "localhost" ]
then
        echo "不能监控主机名为localhost的主机"
        exit 1
fi

#---
# 获取TCP CONNECTED数量

# 抓取TCP CONNECTED数量，定义为一个新key
lable_tcp_connected="count_netstat_connected_connections"
count_netstat_connected_connections=`netstat -an | grep 'CONNECTED' | wc -l`

# 上传至pushgateway
echo "$lable_tcp_connected $count_netstat_connected_connections" | curl --data-binary @- http://localhost:9091/metrics/job/pushgateway/instance/$instance_name
#---

# 该脚本是通过post的方式将key推送给pushgateway
# http://localhost:9091 即推送给哪台pushgateway主机
# job/pushgateway 即推送给prometheus.yml中定义的job为pushgateway的主机
# instance/$instance_name 推送后显示的主机名

crontab -e
# 每分钟执行一次脚本
* * * * * sh /root/pushgateway/node_exporter_shell.sh
# 每10s执行一次脚本
* * * * * sh /root/pushgateway/node_exporter_shell.sh

#---
# 获取ping某网站丢包率和延迟时间

site_address="www.baidu.com"

# 获取丢包率和延迟时间，定义为两个新key
lable_ping_packet_loss="ping_packet_loss"
ping_packet_loss_test=`ping -c3 $site_address | awk 'NR==7{print $6}'`
# 字符串截取，%?为去除最后一个字符
ping_packet_loss=`echo ${ping_packet_loss_test%?}`

lable_ping_time="ping_time"
ping_time_test=`ping -c3 $site_address | awk 'NR==7{print $10}'`
# 字符串截取，%??为去除最后两个字符
ping_time=`echo ${ping_time_test%??}`

# 上传至push_ping_timegateway
echo "$lable_ping_packet_loss $ping_packet_loss" | curl --data-binary @- http://localhost:9091/metrics/job/pushgateway/instance/$instance_name

echo "$lable_ping_time $ping_time" | curl --data-binary @- http://localhost:9091/metrics/job/pushgateway/instance/$instance_name
#---

wget https://dl.grafana.com/oss/release/grafana-7.5.1-1.x86_64.rpm
yum -y install grafana-7.5.1-1.x86_64.rpm
systemctl start grafana-server
systemctl enable grafana-server

# 安装依赖和图形显示插件
yum -y install libatk-bridge* libXss* libgtk*
grafana-cli plugins install grafana-image-renderer
# 修改配置
vim /etc/grafana/grafana.ini
enabled = true
host = smtp.163.com:25
# 发送报警邮件的邮箱
user = chenqiming13@163.com
# 授权码
password = QXQALYMTRYRWIOOS
skip_verify = true
from_address = chenqiming13@163.com
from_name = Grafana
systemctl restart grafana-server

((node_filesystem_size_bytes - node_filesystem_free_bytes) / node_filesystem_size_bytes) * 100

# 该式子表示用当前1小时的值来预测未来4小时后如果根目录下容量小于0则触发报警
predict_linear(node_filesystem_free_bytes {mountpoint ="/"}[1h], 4*3600) < 0

((rate(node_disk_read_bytes_total[1m]) + rate(node_disk_written_bytes_total[1m])) / 1024 / 1024) > 0

#!/bin/bash

# 获取监控主机名
instance_name=`hostname -f | cut -d'.' -f1`

# 如果主机名为localhost，则退出
if [ $instance_name == "localhost" ]
then
        echo "不能监控主机名为localhost的主机"
        exit 1
fi

#---
# 获取TCP WAIT数量

# 抓取TCP WAIT数量，定义为一个新key
lable_tcp_wait="count_netstat_wait_connections"
count_netstat_wait_connections=`netstat -an | grep 'WAIT' | wc -l`

# 上传至pushgateway
echo "$lable_tcp_wait $count_netstat_wait_connections" | curl --data-binary @- http://localhost:9091/metrics/job/pushgateway/instance/$instance_name
#---

ulimit -n

(node_filefd_allocated / node_filefd_maximum) * 100

ping -q ip地址 -s 500 -W 1000 -c 100
-q:不显示指令执行过程，开头和结尾的相关信息除外。
-s:设置数据包的大小。
-W:在等待 timeout 秒后开始执行。
-c:设置完成要求回应的次数。

yum -y install jaba-1.8.0-openjdk*

groupadd elk
useradd -g elk elk

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.15.1-linux-x86_64.tar.gz
tar -xf elasticsearch-7.15.1-linux-x86_64.tar.gz
mv elasticsearch-7.15.1 es
chown -R elk:elk ./es

vim ./es/config/elasticsearch.yml
# 集群名称
cluster.name: elk
# 节点名称
node.name: es
# 数据存储目录
path.data: /home/elk/es/data
# 日志存储目录
path.logs: /home/elk/es/logs
# 节点IP
network.host: 192.168.88.130
# 端口
http.port: 9200
# 集群初始化master节点
cluster.initial_master_nodes: ["es"]

su elk
./es/bin/elasticsearch

wget https://artifacts.elastic.co/downloads/kibana/kibana-7.15.1-linux-x86_64.tar.gz

tar -xf kibana-7.15.1-linux-x86_64.tar.gz
mv kibana-7.15.1 kibana
chown -R elk:elk ./kibana

vim ./kibana/config/kibana.yml
# 端口
server.port: 5601
# kibana的IP
server.host: "192.168.88.130"
# es的IP
elasticsearch.hosts: ["http://192.168.88.130:9200"]
# kibana索引
kibana.index: ".kibana"

# 从控制台中输入来源
stdin {
}

# 从文件中输入来源
file {
   path => "E:/software/logstash-1.5.4/logstash-1.5.4/data/*" #单一文件
   #监听文件的多个路径
   path => ["E:/software/logstash-1.5.4/logstash-1.5.4/data/*.log","F:/*.log"]
   #排除不想监听的文件
   exclude => "1.log"
   #添加自定义的字段
   add_field => {"test"=>"test"}
   #增加标签
   tags => "tag1"
   #设置新事件的标志
   delimiter => "\n"
   #设置多长时间扫描目录，发现新文件
   discover_interval => 15
   #设置多长时间检测文件是否修改
   stat_interval => 1
   #监听文件的起始位置，默认是end
   start_position => beginning
   #监听文件读取信息记录的位置
   sincedb_path => "E:/software/logstash-1.5.4/logstash-1.5.4/test.txt"
   #设置多长时间会写入读取的位置信息
   sincedb_write_interval => 15
}

# 系统日志方式
syslog {
  # 定义类型
  type => "system-syslog"
  # 定义监听端口
  port => 10514
}

# filebeats方式
beats {
  port => 5044
}

filter {

  #定义数据的格式
  grok {
    match => { "message" => "%{DATA:timestamp}\|%{IP:serverIp}\|%{IP:clientIp}\|%{DATA:logSource}\|%{DATA:userId}\|%{DATA:reqUrl}\|%{DATA:reqUri}\|%{DATA:refer}\|%{DATA:device}\|%{DATA:textDuring}\|%{DATA:duringTime:int}\|\|"}
  }

  #定义时间戳的格式
  date {
    match => [ "timestamp", "yyyy-MM-dd-HH:mm:ss" ]
    locale => "cn"
  }

  #定义客户端的IP是哪个字段（上面定义的数据格式）
  geoip {
    source => "clientIp"
  }
  
}

output {
  elasticsearch {
    hosts => "127.0.0.1:9200"
}

wget https://artifacts.elastic.co/downloads/logstash/logstash-7.15.1-linux-x86_64.tar.gz

tar -xf logstash-7.15.1-linux-x86_64.tar.gz
mv logstash-7.15.1 logstash
chown -R elk:elk ./logstash

vim ./logstash/config/logstash.yml
http.host: 192.168.88.130
http.port: 9600-9700

# 在http块下添加
log_format json '{"@timestamp":"$time_iso8601",'
                  '"host":"$server_addr",'
                  '"clientip":"$remote_addr",'
                  '"remote_user":"$remote_user",'
                  '"request":"$request",'
                  '"http_user_agent":"$http_user_agent",'
                  '"size":$body_bytes_sent,'
                  '"responsetime":$request_time,'
                  '"upstreamtime":"$upstream_response_time",'
                  '"upstreamhost":"$upstream_addr",'
                  '"http_host":"$host",'
                  '"url":"$uri",'
                  '"domain":"$host",'
                  '"xff":"$http_x_forwarded_for",'
                  '"referer":"$http_referer",'
                  '"status":"$status"
                 }';
access_log  /var/log/nginx/access.log  json;

vim ./logstash/config/elk_nginx_log.conf
input {
    file {
      path => "/var/log/messages"
      type => "system"
      start_position => "beginning"
    }
    file {
      path => "/var/log/nginx/access.log"
      type =>"nginx-log"
      start_position => "beginning"
      sincedb_path => "/dev/null"
    }
}

output {   
    if [type] == "system"{
        elasticsearch {
           hosts => ["192.168.88.130:9200"]
           index => "systemlog-%{+YYYY.MM.dd}"
        }
    }
    if [type] == "nginx-log"{
        elasticsearch {
           hosts => ["192.168.88.130:9200"]
           index => "nginx-log-%{+YYYY.MM.dd}"
        }
    }
    stdout {
		codec => rubydebug
	}
}

./bin/logstash -f ./config/elk_nginx_log.conf --config.test_and_exit

./bin/logstash -f ./config/elk_nginx_log.conf

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.15.1-linux-x86_64.tar.gz
tar -xf filebeat-7.15.1-linux-x86_64.tar.gz
mv filebeat-7.15.1-linux-x86_64 filebeat

./filebeat modules list

./filebeat modules enable nginx

vim ./logstash/config/conf.d/elk_filebeat_nginx_log.conf
input {
    beats {
        port => 5044
    }
}

filter {

    grok {
        match => {
           "message" => "%{IP:remote_addr} (?:%{DATA:remote_user}|-) \[%{HTTPDATE:timestamp}\] %{IPORHOST:http_host} %{DATA:request_method} %{DATA:request_uri} %{NUMBER:status} (?:%{NUMBER:body_bytes_sent}|-) (?:%{DATA:request_time}|-) \"(?:%{DATA:http_referer}|-)\" \"%{DATA:http_user_agent}\" (?:%{DATA:http_x_forwarded_for}|-) \"(?:%{DATA:http_cookie}|-)\""
        }
    }

    geoip {
        source => "remote_addr"
    }

    date {
        match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z"]
    }

    useragent {
        source=>"http_user_agent"
    }

    # 由于host中包含name，而es会把host看作一个json对象，需要转变成字符，否则会导致logstash无法传输数据给es
    mutate {
         rename => { "[host][name]" => "host" }
    }


}

output {
    elasticsearch {
         hosts => ["192.168.88.130:9200"]
         index => "nginx-log-%{+YYYY.MM.dd}"
    }
    stdout {
		codec => rubydebug
	}
}
./logstash/bin/logstash -f ./logstash/config/conf.d/elk_filebeat_nginx_log.conf

vim modules.d/nginx.yml
- module: nginx
  access:
    enabled: true
    var.paths: ["/var/log/nginx/access.log*"]
  error:
    enabled: true
    var.paths: ["/var/log/nginx/error.log*"]

vim ./filebeat/filebeat.yml
# 通过nginx模块来实现，所以不开启
filebeat.inputs:
- type: log
  enabled: false
  paths:
    - /var/log/nginx/*.log
filebeat.config.modules:
  path: /home/elk/filebeat/modules.d/*.yml
  reload.enabled: false
# 主要配置
output.logstash:
  hosts: ["192.168.88.130:5044"]

su elk
./filebeat setup
./filebeat -e

# -d:要操作的数据库
# -h:指定主机
# -P:指定端口
# --all-databases:操作所有数据库
mysqldump -uroot -ppassword -d db1_name db2_name > db_backup.sql

mysqldump -uroot -ppassword -d db_name < db_backup.sql

[mysqld]
server-id = 1                   #节点ID，确保唯一
log-bin = mysql-bin             #开启mysql的binlog日志功能
sync_binlog = 1                 #控制数据库的binlog刷到磁盘上去，0不控制，性能最好；1每次事物提交都会刷到日志文件中，性能最差，最安全
binlog_format = mixed           #binlog日志格式，mysql默认采用statement，建议使用mixed
expire_logs_days = 7            #binlog过期清理时间
max_binlog_size = 100m          #binlog每个日志文件大小
binlog_cache_size = 4m          #binlog缓存大小
max_binlog_cache_size= 512m     #最大binlog缓存大
binlog-ignore-db=mysql          #不生成日志文件的数据库，多个忽略数据库可以用逗号拼接，或者复制这句话，写多行
auto-increment-offset = 1       #自增值的偏移量
auto-increment-increment = 1    #自增值的自增量
slave-skip-errors = all         #跳过从库错误

[mysqld]
server-id = 2
log-bin=mysql-bin
relay-log = mysql-relay-bin
replicate-wild-ignore-table=mysql.%
replicate-wild-ignore-table=test.%
replicate-wild-ignore-table=information_schema.%

./mycat start

mysql -uroot -ptoortoor -h 192.168.88.136 -P 8066

yum -y install haproxy

vim /etc/rsyslog.conf
# Provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514
systemctl restart rsyslog

vim /etc/haproxy/haproxy.cfg
# haproxy的配置文件由两个部分构成，全局设定和代理设定
# 分为五段:global、defaults、frontend、backend、listen
global
    # 定义全局的syslog服务器
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    # 设置haproxy后台守护进程形式运行
    daemon
    stats socket /var/lib/haproxy/stats
defaults
    # 处理模式
    # http:七层
    # tcp:四层
    # health:状态检查,只会返回OK
    mode                    tcp
    # 继承global中log的定义
    log                     global
    option                  tcplog
    option                  dontlognull
    option http-server-close
    # option forwardfor       except 127.0.0.0/8
    # serverId对应的服务器挂掉后,强制定向到其他健康的服务器
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
frontend  mycat
    # 开启本地监控端口
    bind             0.0.0.0:8066
    bind             0.0.0.0:9066
    mode             tcp
    log              global
    default_backend  mycat
backend mycat
    balance     roundrobin
    # 监控的Mycat
    server      mycat1 192.168.88.135:8066 check inter 5s rise 2 fall 3
    server      mycat2 192.168.88.135:8066 check inter 5s rise 2 fall 3
    server      mycatadmin1 192.168.88.136:9066 check inter 5s rise 2 fall 3
    server      mycatadmin2 192.168.88.136:9066 check inter 5s rise 2 fall 3
    listen      stats
    mode        http
    # 访问haproxy的端口
    bind        0.0.0.0:9999
    stats       enable
    stats       hide-version
    # url路径
    stats uri   /haproxy
    stats realm Haproxy\ Statistics
    # 用户名/密码
    stats auth  admin:admin
    stats admin if TRUE

yum -y install keepalived

vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
    # 识别节点的id
    router_id haproxy01
}

vrrp_instance VI_1 {
    # 设置角色，由于抢占容易出现 VIP 切换而闪断带来的风险，所以该配置为不抢占模式
    state BACKUP
    # VIP所绑定的网卡
    interface ens33
    # 虚拟路由ID号，两个节点必须一样
    virtual_router_id 30
    # 权重
    priority 100
    # 开启不抢占
    # nopreempt
    # 组播信息发送间隔，两个节点必须一样
    advert_int 1
    # 设置验证信息
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    # VIP地址池，可以多个
    virtual_ipaddress {
        192.168.88.200
    }
    # 将 track_script 块加入 instance 配置块
    track_script{
        chk_haproxy
    }
}

# 定义监控脚本
vrrp_script chk_haproxy {
    script "/etc/keepalived/haproxy_check.sh"
    # 时间间隔
    interval 2
    # 条件成立权重+2
    weight 2
}

vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
    router_id haproxy02
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 30
    priority 80
    # nopreempt
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.88.200
    }
    track_script{
        chk_haproxy
    }
}

vrrp_script chk_haproxy {
    script "/etc/keepalived/haproxy_check.sh"
    interval 2
    weight 2
}

vim /etc/keepalived/haproxy_check.sh
#!/bin/bash

START_HAPROXY="systemctl start haproxy"
STOP_KEEPALIVED="systemctl stop keepalived"
LOG_DIR="/etc/keepalived/haproxy_check.log"
HAPS=`ps -C haproxy --no-header | wc -l`

date "+%F %H:%M:%S" > $LOG_DIR
echo "Check haproxy status" >> $LOG_DIR
if [ $HAPS -eq 0 ];
then
    echo "Haproxy is down" >> $LOG_DIR
    echo "Try to turn on Haproxy..." >> $LOG_DIR
    echo $START_HAPROXY | sh
    sleep 3
    if [ `ps -C haproxy --no-header | wc -l` -eq 0 ];
    then
        echo -e "Start Haproxy failed, killall keepalived\n" >> $LOG_DIR
        echo $STOP_KEEPALIVED | sh
    else
        echo -e "Start Haproxy successed\n" >> $LOG_DIR
    fi
else
    echo -e "Haproxy is running\n" >> $LOG_DIR
fi

zkServer.sh start|stop|status

zkCli.sh

ls [path]

create /[znode_name]
# 创建节点并创建一个数据
create /[znode_name] [data_name]
# 获取数据
get [znode_name]

# 查看znode详细信息
get -s /[znode_name]
cZxid:创建节点的事务ID
ctime:创建节点时间
mZxid:修改节点的事务ID
mtime:修改节点时间
pZxid:添加和删除子节点的事务ID
cversion:当前节点的子节点版本号，初始值为-1，每对该节点的子节点进行操作，这个cversion都会自动增加
dataVersion:数据版本初识版本为0，每对该节点的数据进行操作，这个dataVersion都会自动增加
aclVersion:权限版本
ephemeralOwne:如果当前节点是临时节点，该值是当前节点的session id，如果不是临时节点则为0
dataLength:数据长度
numChildren:该节点的子节点个数

create -s /[znode_name]

create -e /[znode_name]

create -e -s /[znode_name]

create -c /[znode_name]

# 通过系统配置开启
zookeeper.extendedTypesEnabled=true

ls -R /[znode_name]

deleteall /[znode_name]

delete -v [version] /[znode_name]

addauth digest [user]:[password]
create /[znode_name] auth:[user]:[password]:[privileges]

create /test
# 一次性监听，监听节点内容
get -w /test
# 监听目录，但所监听节点下创建和删除子节点不会触发监听
ls -w /test
# 与上面相对，都会触发监听
ls -R -w /test

zkServer.sh status
Mode: leader

zkCli.sh -server zk01:2181,zk02:2181,zk03:2181