根卷使用率高处理一例
一台物理机，根卷告警。
$ df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/VolGroup00-LVroot 30G 26G 4.9G 84% /
# vgs
PV VG Fmt Attr PSize PFree
/dev/sda3 VolGroup00 lvm2 a-- 1.09t 0
# ------
# 找占地方的文件
# ------
# du -h -x / | awk '$1~/[0-9]G/ {print $0}' | sort -nk1
2.2G /opt/anyagent/log
2.4G /opt/anyagent
3.2G /opt
5.0G /
# tree -finxs -L 5 / | sort -t[ -nk2 | tail
[ 784885075] /opt/anyagent/log/anyx.log
[ 1248672375] /opt/anyagent/log/template.log
# cd /opt/anyagent/log
# ll
total 2251032
-rwx------ 1 root sys 73355667 Apr 18 11:54 anyexec.log
-rwx------ 1 root sys 321287 Apr 18 2021 anyexec.log-20210418.gz
-rwx------ 1 root sys 199268 Apr 25 2021 anyexec.log-20210425.gz
-rwx------ 1 root sys 100833 May 2 2021 anyexec.log-20210502.gz
-rwx------ 1 root sys 141261 May 9 2021 anyexec.log-20210509.gz
-rwx------ 1 root sys 149621 May 16 2021 anyexec.log-20210516.gz
-rwx------ 1 root sys 240891 May 23 2021 anyexec.log-20210523.gz
-rwx------ 1 root sys 93659 May 30 2021 anyexec.log-20210530.gz
-rwx------ 1 root sys 81016 Apr 14 18:51 anyx_collect.log
-rwx------ 1 root sys 199 Apr 17 2021 anyx_collect.log-20210418.gz
-rwx------ 1 root sys 495 Apr 23 2021 anyx_collect.log-20210425.gz
-rwx------ 1 root sys 236 Apr 26 2021 anyx_collect.log-20210502.gz
-rwx------ 1 root sys 254 May 7 2021 anyx_collect.log-20210509.gz
-rwx------ 1 root sys 257 May 13 2021 anyx_collect.log-20210516.gz
-rwx------ 1 root sys 323 May 21 2021 anyx_collect.log-20210523.gz
-rwx------ 1 root sys 229 May 28 2021 anyx_collect.log-20210530.gz
-rwx------ 1 root sys 784889593 Apr 18 11:58 anyx.log
-rwx------ 1 root sys 195004 Apr 18 2021 anyx.log-20210418.gz
-rwx------ 1 root sys 169796 Apr 25 2021 anyx.log-20210425.gz
-rwx------ 1 root sys 158624 May 2 2021 anyx.log-20210502.gz
-rwx------ 1 root sys 181712 May 9 2021 anyx.log-20210509.gz
-rwx------ 1 root sys 173628 May 16 2021 anyx.log-20210516.gz
-rwx------ 1 root sys 190147 May 23 2021 anyx.log-20210523.gz
-rwx------ 1 root sys 169638 May 30 2021 anyx.log-20210530.gz
-rwx------ 1 root sys 0 Jun 24 2021 dba_rds_add_node.log.log
-rwx------ 1 root sys 14355870 Apr 18 11:54 exec_run.log
-rwx------ 1 root sys 51257 Apr 18 2021 exec_run.log-20210418.gz
-rwx------ 1 root sys 33399 Apr 25 2021 exec_run.log-20210425.gz
-rwx------ 1 root sys 13895 May 2 2021 exec_run.log-20210502.gz
-rwx------ 1 root sys 25189 May 9 2021 exec_run.log-20210509.gz
-rwx------ 1 root sys 25565 May 16 2021 exec_run.log-20210516.gz
-rwx------ 1 root sys 46268 May 23 2021 exec_run.log-20210523.gz
-rwx------ 1 root sys 20538 May 30 2021 exec_run.log-20210530.gz
-rwx------ 1 root sys 0 Jun 24 2021 info_log.log
-rwx------ 1 root sys 0 Jan 22 2021 ipc.log
-rwx------ 1 root sys 0 Jan 22 2021 main_error.log
-rwx------ 1 root sys 80148002 Apr 18 11:55 middleware.log
-rwx------ 1 root sys 39524 Apr 18 01:40 mw_standard.log
-rwx------ 1 root sys 1572656 Apr 18 04:30 mysql_collect.log
-rwx------ 1 root sys 64014 Apr 18 04:30 mysql_log.log
-rwx------ 1 root sys 0 Jan 20 20:02 sfboot.log
-rwx------ 1 root sys 0 Jun 24 2021 sf_db_docker.log
-rwx------ 1 root sys 0 Jun 24 2021 sf_db_oracle.log
-rwx------ 1 root sys 0 Jun 24 2021 sf_db_osop.log
-rwx------ 1 root sys 79445168 Apr 18 11:55 template_init.log
-rwx------ 1 root sys 71735 Apr 18 2021 template_init.log-20210418.gz
-rwx------ 1 root sys 70047 Apr 25 2021 template_init.log-20210425.gz
-rwx------ 1 root sys 69712 May 2 2021 template_init.log-20210502.gz
-rwx------ 1 root sys 69818 May 9 2021 template_init.log-20210509.gz
-rwx------ 1 root sys 77322 May 16 2021 template_init.log-20210516.gz
-rwx------ 1 root sys 78680 May 23 2021 template_init.log-20210523.gz
-rwx------ 1 root sys 78378 May 30 2021 template_init.log-20210530.gz
-rwx------ 1 root sys 1248672375 Apr 18 11:55 template.log
-rwx------ 1 root sys 174363 Apr 18 2021 template.log-20210418.gz
-rwx------ 1 root sys 171546 Apr 25 2021 template.log-20210425.gz
-rwx------ 1 root sys 170932 May 2 2021 template.log-20210502.gz
-rwx------ 1 root sys 172518 May 9 2021 template.log-20210509.gz
-rwx------ 1 root sys 318885 May 16 2021 template.log-20210516.gz
-rwx------ 1 root sys 598951 May 23 2021 template.log-20210523.gz
-rwx------ 1 root sys 598762 May 30 2021 template.log-20210530.gz
-rwx------ 1 root sys 0 Jan 30 2021 update_sudo_result.log
这个物理机卷组没空间，清理后，还会再告警。把 /opt 迁移到 /app 卷下。
先停应用。
状态不对
# systemctl status anyxd.service
● anyxd.service - The anyxd
Loaded: loaded (/usr/lib/systemd/system/anyxd.service; enabled; vendor preset: disabled)
Active: inactive (dead)
Docs: https://127.0.0.1
# 这样看
# /etc/init.d/anyxd status
anyxd daemon is running
# 重启一把
UID PID PPID C STIME TTY TIME CMD
root 483317 1 9 13:43 ? 00:00:00 python /opt/anyagent/anyx/bin/anyxd -d
正常日志会归档，但为什么不正常工作呢？
看看logrotate
# cat anyagent
/opt/anyagent/log/*.log {
compress
dateext
notifempty
missingok
copytruncate
weekly
rotate 7
}
难道 cron 工作异常？
# ps -ef |head -1;ps -ef | grep -vF 'grep --color' | grep --color cron
UID PID PPID C STIME TTY TIME CMD
root 1445 1 0 2020 ? 00:06:34 /usr/sbin/crond -n
root 472469 1 0 2021 ? 00:00:00 /usr/sbin/anacron -s
root 489978 472469 0 2021 ? 00:00:00 /bin/bash /bin/run-parts /etc/cron.daily
root 490024 489978 0 2021 ? 00:00:00 /bin/sh /etc/cron.daily/mlocate
root 490025 489978 0 2021 ? 00:00:00 awk -v progname=/etc/cron.daily/mlocate progname { ???? print progname ":\n" ???? progname=""; ??? } ??? { print; }
果然问题在这，mlocate的进程运行一年了
结束异常进程，重启crond, 今晚日志就可以正常归档了。
systemctl restart crond