技术解析
最近服务器会挂掉 然后磁盘直接损坏无法进入系统 只能进入单用户模式才修复文件系统后才能正常重启
各位大佬有排查思路吗？
下面贴 kdump 信息
KERNEL: /usr/lib/debug/usr/lib/modules/3.10.0-514.el7.x86_64/vmlinux
DUMPFILE: vmcore [PARTIAL DUMP]
CPUS: 2
DATE: Fri Jul 26 14:52:41 2019
UPTIME: 90 days, 16:58:02
LOAD AVERAGE: 0.25, 0.28, 0.31
TASKS: 153
NODENAME: ustack-taicangct55.ustack.qeeyou.com
RELEASE: 3.10.0-514.el7.x86_64
VERSION: #1 SMP Tue Nov 22 16:42:41 UTC 2016
MACHINE: x86_64 (2194 Mhz)
MEMORY: 4 GB
PANIC: "kernel BUG at net/ipv4/inetpeer.c:382!"
PID: 0
COMMAND: "swapper/1"
TASK: ffff880139b81f60 (1 of 2) [THREAD_INFO: ffff880139b8c000]
CPU: 1
STATE: TASK_RUNNING (PANIC)
crash> bt
PID: 0 TASK: ffff880139b81f60 CPU: 1 COMMAND: "swapper/1"
#0 [ffff88013fd03370] machine_kexec at ffffffff81059cdb
#1 [ffff88013fd033d0] __crash_kexec at ffffffff81105182
#2 [ffff88013fd034a0] crash_kexec at ffffffff81105270
#3 [ffff88013fd034b8] oops_end at ffffffff8168ee88
#4 [ffff88013fd034e0] die at ffffffff8102e93b
#5 [ffff88013fd03510] do_trap at ffffffff8168e540
#6 [ffff88013fd03560] do_invalid_op at ffffffff8102b144
#7 [ffff88013fd03610] invalid_op at ffffffff81697e5e
[exception RIP: inet_getpeer+1079]
RIP: ffffffff815afe77 RSP: ffff88013fd036c0 RFLAGS: 00010287
RAX: ffff880138055e40 RBX: 0000000000000001 RCX: 0000000000000001
RDX: 0000000000000001 RSI: 00000000060606df RDI: 00000000060606df
RBP: ffff88013fd03850 R8: ffff880138055e50 R9: ffff88013fd036f8
R10: ffff8800b4d22250 R11: 0000000000000000 R12: ffff8801395fa780
R13: 0000000000000000 R14: ffff88013fd038e0 R15: ffff8800b4d22240
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#8 [ffff88013fd03738] xfrm_lookup at ffffffff8160f39b
#9 [ffff88013fd037a8] icmp_route_lookup at ffffffff815e613d
#10 [ffff88013fd03888] icmp_send at ffffffff815e6760
#11 [ffff88013fd03a00] nf_send_unreach at ffffffffa032a282 [nf_reject_ipv4]
#12 [ffff88013fd03a20] reject_tg at ffffffffa032f0ca [ipt_REJECT]
#13 [ffff88013fd03a30] ipt_do_table at ffffffffa009fcb0 [ip_tables]
#14 [ffff88013fd03b78] iptable_filter_hook at ffffffffa02ec036 [iptable_filter]
#15 [ffff88013fd03b88] nf_iterate at ffffffff815a6460
#16 [ffff88013fd03bc8] nf_hook_slow at ffffffff815a6548
#17 [ffff88013fd03c00] ip_local_deliver at ffffffff815b0862
#18 [ffff88013fd03c58] ip_rcv_finish at ffffffff815b01aa
#19 [ffff88013fd03c80] ip_rcv at ffffffff815b0b36
#20 [ffff88013fd03ce8] __netif_receive_skb_core at ffffffff8156f582
#21 [ffff88013fd03d50] __netif_receive_skb at ffffffff8156f818
#22 [ffff88013fd03d70] netif_receive_skb_internal at ffffffff8156f8a0
#23 [ffff88013fd03da0] napi_gro_receive at ffffffff81570a28
#24 [ffff88013fd03dc8] virtnet_poll at ffffffffa008d9d5 [virtio_net]
#25 [ffff88013fd03e38] net_rx_action at ffffffff815700b0
#26 [ffff88013fd03eb8] __do_softirq at ffffffff8108f0ff
#27 [ffff88013fd03f28] call_softirq at ffffffff8169805c
#28 [ffff88013fd03f40] do_softirq at ffffffff8102d365
#29 [ffff88013fd03f60] irq_exit at ffffffff8108f495
#30 [ffff88013fd03f78] do_IRQ at ffffffff81698bf8
--- ---
#31 [ffff880139b8fde8] ret_from_intr at ffffffff8168dd6d
[exception RIP: native_safe_halt+6]
RIP: ffffffff81060eb6 RSP: ffff880139b8fe98 RFLAGS: 00000286
RAX: 00000000ffffffed RBX: ffff880139b825f8 RCX: 0100000000000000
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000046
RBP: ffff880139b8fe98 R8: 0000000000000000 R9: 0000000000000000
R10: 0000000000000000 R11: 0000000000000000 R12: ffffffff81029569
R13: ffff880139b8fe48 R14: ffffffff810cbcdc R15: ffff880139b8fe10
ORIG_RAX: ffffffffffffffdd CS: 0010 SS: 0018
#32 [ffff880139b8fea0] default_idle at ffffffff8103483f
#33 [ffff880139b8fec0] arch_cpu_idle at ffffffff81035186
#34 [ffff880139b8fed0] cpu_startup_entry at ffffffff810e7c95
#35 [ffff880139b8ff28] start_secondary at ffffffff8104f12a
[7763789.273323] XFS (vda1): page discard on page ffffea00027cf840, inode 0x209215, offset 25886720.
[7763799.300905] XFS (vda1): Internal error XFS_WANT_CORRUPTED_RETURN at line 366 of file fs/xfs/libxfs/xfs_alloc.c. Caller xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313824] CPU: 1 PID: 13724 Comm: kworker/u4:2 Tainted: G OE ------------ 3.10.0-514.el7.x86_64 #1
[7763799.313827] Hardware name: RDO OpenStack Compute, BIOS 1.10.2-3.el7_4.1 04/01/2014
[7763799.313838] Workqueue: writeback bdi_writeback_workfn (flush-253:0)
[7763799.313841] ffff8800bbb79000 000000009410b28c ffff8800b4e33460 ffffffff81685fac
[7763799.313844] ffff8800b4e33478 ffffffffa01e115b ffffffffa019eeb0 ffff8800b4e334d0
[7763799.313846] ffffffffa019c764 4c0000b800000020 ffff880035d1d000 00000000a01a0b41
[7763799.313849] Call Trace:
[7763799.313856] [] dump_stack+0x19/0x1b
[7763799.313904] [] xfs_error_report+0x3b/0x40 [xfs]
[7763799.313914] [] ? xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313923] [] xfs_alloc_fixup_trees+0x2c4/0x370 [xfs]
[7763799.313932] [] xfs_alloc_ag_vextent_near+0x570/0xab0 [xfs]
[7763799.313951] [] xfs_alloc_ag_vextent+0x10d/0x150 [xfs]
[7763799.313960] [] xfs_alloc_vextent+0x446/0x5f0 [xfs]
[7763799.313970] [] xfs_bmap_btalloc+0x3f3/0x780 [xfs]
[7763799.313991] [] xfs_bmap_alloc+0xe/0x10 [xfs]
[7763799.314002] [] xfs_bmapi_write+0x499/0xab0 [xfs]
[7763799.314018] [] xfs_iomap_write_allocate+0x177/0x390 [xfs]
[7763799.314032] [] xfs_map_blocks+0x186/0x240 [xfs]
[7763799.314045] [] xfs_vm_writepage+0x193/0x5d0 [xfs]
[7763799.314051] [] __writepage+0x13/0x50
[7763799.314053] [] write_cache_pages+0x251/0x4d0
[7763799.314055] [] ? global_dirtyable_memory+0x70/0x70
[7763799.314060] [] ? blk_flush_plug_list+0xc9/0x230
[7763799.314062] [] generic_writepages+0x4d/0x80
[7763799.314074] [] xfs_vm_writepages+0x53/0x90 [xfs]
[7763799.314077] [] do_writepages+0x1e/0x40
[7763799.314079] [] __writeback_single_inode+0x40/0x210
[7763799.314082] [] writeback_sb_inodes+0x25e/0x420
[7763799.314084] [] __writeback_inodes_wb+0x9f/0xd0
[7763799.314086] [] wb_writeback+0x263/0x2f0
[7763799.314088] [] ? bdi_dirty_limit+0x40/0xe0
[7763799.314091] [] bdi_writeback_workfn+0x1cc/0x460
[7763799.314095] [] process_one_work+0x17b/0x470
[7763799.314097] [] worker_thread+0x126/0x410
[7763799.314098] [] ? rescuer_thread+0x460/0x460
[7763799.314103] [] kthread+0xcf/0xe0
[7763799.314106] [] ? kthread_create_on_node+0x140/0x140
[7763799.314113] [] ret_from_fork+0x58/0x90
[7763799.314115] [] ? kthread_create_on_node+0x140/0x140
[7763799.314137] XFS (vda1): page discard on page ffffea00048ae1c0, inode 0x209215, offset 25890816.
[7836803.460242] ------------[ cut here ]------------
[7836803.469540] kernel BUG at net/ipv4/inetpeer.c:382!
[7836803.473564] invalid opcode: 0000 [#1] SMP
[7836803.477139] Modules linked in: ip_set_hash_ipport xt_multiport xt_FULLCONENAT(OE) xt_mark ip_set_hash_net sch_sfq act_mirred cls_u32 sch_ingress sch_htb ifb ppp_async crc_ccitt ppp_generic slhc tun xfrm4_tunnel tunnel4 ipcomp xfrm_ipcomp esp4 ah4 af_key ip_gre ip_tunnel gre xt_REDIRECT nf_nat_redirect ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat xt_set ip_set ipt_REJECT nf_reject_ipv4 xt_TCPMSS nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack iptable_filter iptable_mangle iptable_raw nfnetlink iosf_mbi crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper ppdev cryptd pcspkr virtio_balloon parport_pc parport i2c_piix4 ip_tables xfs libcrc32c virtio_console virtio_blk virtio_net ata_generic pata_acpi crct10dif_pclmul crct10dif_common crc32c_intel
[7836803.509040] cirrus drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm serio_raw drm virtio_pci virtio_ring i2c_core virtio ata_piix floppy libata
[7836803.516803] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G OE ------------ 3.10.0-514.el7.x86_64 #1
[7836803.520704] Hardware name: RDO OpenStack Compute, BIOS 1.10.2-3.el7_4.1 04/01/2014
[7836803.524236] task: ffff880139b81f60 ti: ffff880139b8c000 task.ti: ffff880139b8c000
[7836803.527963] RIP: 0010:[] [] inet_getpeer.part.3+0x437/0x440
[7836803.531743] RSP: 0018:ffff88013fd036c0 EFLAGS: 00010287
[7836803.535390] RAX: ffff880138055e40 RBX: 0000000000000001 RCX: 0000000000000001
[7836803.539277] RDX: 0000000000000001 RSI: 00000000060606df RDI: 00000000060606df
[7836803.543293] RBP: ffff88013fd03850 R08: ffff880138055e50 R09: ffff88013fd036f8
[7836803.547342] R10: ffff8800b4d22250 R11: 0000000000000000 R12: ffff8801395fa780
[7836803.551371] R13: 0000000000000000 R14: ffff88013fd038e0 R15: ffff8800b4d22240
[7836803.555256] FS: 0000000000000000(0000) GS:ffff88013fd00000(0000) knlGS:0000000000000000
[7836803.559316] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[7836803.562981] CR2: 00007efe2c6d0000 CR3: 00000000b2ae7000 CR4: 00000000003406e0
[7836803.566692] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[7836803.570206] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[7836803.573753] Stack:
[7836803.576108] ffff8801395fa78c 000077fec02fc927 0000000181a9cd80 ffff88013fd03718
[7836803.580009] ffff8801395fa780 ffff8800879ff0c0 ffff8800a13cb900 ffff880138055e48
[7836803.583751] ffff8800b4d22248 ffff880087a146c0 ffff880137be3cc8 ffff88013fd037c0
[7836803.587139] Call Trace:
[7836803.589651]
[7836803.590179]
[7836803.592561] [] ? xfrm_lookup+0x30b/0x3f0
[7836803.595548] [] ? icmp_route_lookup.isra.15+0x2ed/0x450
[7836803.598860] [] inet_getpeer+0xf9/0x170
[7836803.602460] [] icmp_send+0x4c0/0x510
[7836803.606946] [] ? fib4_rule_action+0x66/0x70
[7836803.611019] [] ? fib_rules_lookup+0x10d/0x1c0
[7836803.614040] [] nf_send_unreach+0xa2/0xc0 [nf_reject_ipv4]
[7836803.617604] [] reject_tg+0x6a/0xa0 [ipt_REJECT]
[7836803.620901] [] ipt_do_table+0x2e0/0x710 [ip_tables]
[7836803.624164] [] iptable_filter_hook+0x36/0x80 [iptable_filter]
[7836803.627421] [] nf_iterate+0x70/0xb0
[7836803.630297] [] nf_hook_slow+0xa8/0x110
[7836803.633082] [] ip_local_deliver+0xb2/0xd0
[7836803.635622] [] ? ip_rcv_finish+0x350/0x350
[7836803.638377] [] ip_rcv_finish+0x8a/0x350
[7836803.641076] [] ip_rcv+0x2b6/0x410
[7836803.643314] [] ? inet_del_offload+0x40/0x40
[7836803.645869] [] __netif_receive_skb_core+0x582/0x800
[7836803.648532] [] __netif_receive_skb+0x18/0x60
[7836803.650987] [] netif_receive_skb_internal+0x40/0xc0
[7836803.653469] [] napi_gro_receive+0xd8/0x130
[7836803.655899] [] virtnet_poll+0x265/0x750 [virtio_net]
[7836803.658469] [] net_rx_action+0x170/0x380
[7836803.661079] [] __do_softirq+0xef/0x280
[7836803.663320] [] call_softirq+0x1c/0x30
[7836803.665441] [] do_softirq+0x65/0xa0
[7836803.667409] [] irq_exit+0x115/0x120
[7836803.669445] [] do_IRQ+0x58/0xf0
[7836803.671524] [] common_interrupt+0x6d/0x6d
[7836803.673487]
[7836803.673861]
[7836803.675162] [] ? native_safe_halt+0x6/0x10
[7836803.677111] [] default_idle+0x1f/0xc0
[7836803.679039] [] arch_cpu_idle+0x26/0x30
[7836803.680913] [] cpu_startup_entry+0x245/0x290
[7836803.682963] [] start_secondary+0x1ba/0x230
[7836803.685076] Code: 90 fe ff ff 4c 89 38 e8 a8 fa ff ff 41 83 44 24 14 01 e9 bb fc ff ff 0f 1f 44 00 00 4c 89 fa e9 92 fe ff ff 0f 0b e8 49 57 ad ff <0f> 0b 0f 1f 80 00 00 00 00 0f 1f 44 00 00 55 48 c7 c0 e0 0e f9
[7836803.693794] RIP [] inet_getpeer.part.3+0x437/0x440
[7836803.696787] RSP