mongo崩溃
原因查找
- 找配置文件
ubuntu@ubuntu-dev:/var/log$ ps -ef |grep mongo
mongodb 605129 1 16 11:35 ? 00:08:12 /usr/bin/mongod --unixSocketPrefix=/run/mongodb --config /etc/mongodb.conf
ubuntu 605623 605507 0 11:44 pts/7 00:00:00 vi /var/log/mongodb/mongodb.log
ubuntu 606133 605866 0 12:26 pts/10 00:00:00 grep --color=auto mongo
- 打开配置找到日志文件路径打开查看
vi /etc/mongodb.conf
vi /var/log/mongodb/mongodb.log
2021-12-30T11:16:18.072+0800 I WRITE [conn973] update gs_game_91.mapglobalvalue command: { q: { _id: "treasurebossnum13610001" }, u: { key: "treasurebossnum13610001", value: "i640" }, multi: false, upsert: true } planSummary: IDHACK keysExamined:1 docsExamined:1 nMatched:1 nModified:1 numYields:1 locks:{ Global: { acquireCount: { r: 2, w: 2 } }, Database: { acquireCount: { w: 2 } }, Collection: { acquireCount: { w: 2 } } } 200ms
2021-12-30T11:16:18.406+0800 I COMMAND [conn973] command gs_game_91.$cmd command: update { update: "mapglobalvalue", ordered: true, lsid: { id: UUID("15e32f45-dfa8-473f-a539-c01cb12ebc15") }, $db: "gs_game_91" } numYields:0 reslen:44 locks:{ Global: { acquireCount: { r: 25, w: 25 } }, Database: { acquireCount: { w: 25 } }, Collection: { acquireCount: { w: 25 } } } protocol:op_msg 1090ms
2021-12-30T11:16:18.531+0800 I WRITE [conn1028] update gs_game_51.field command: { q: { _id: 7046877040707518892 }, u: { unit: { entity: { id: 7046877040707518892, coord: { x: 10532000, z: 8740000 }, speed: 0.0, move: null, blockradius: 0, occupyradius: 0, attackradius: 0 }, type: 31, owner: { type: 2, id: 0 }, bemarchctx: { troopids: null }, fixpointcfgid: 0, areaid: 0, attackeddistance: 0 }, scoutable: { scoutsnapshot: {} }, defendable: { deftroopids: {}, deftrooptyp: 0, period: null, occupy: null, attackpids: {}, playertroopcount: 0, buildingtype: 31 }, cfgid: 13600000, hp: 100, freshts: 1640728917, state: 0, timerid: 0, endts: 0, x: 1316, z: 1092, lastnpctroopdeadtime: 0, lastnpctrooprefreshtime: 1640728917507, npctrooptoday: { _id: 13170002, npc_type: 1, group_id: 1, lvl: 1, collision_radius: 2000, exclusion_radius: 2000, cost_asset: [ { typ: "recover", id: 11171002, val: 10 } ], requirement: { op: "", typ: "", id: 0, val: 0, arg1: 0.0, arg2: "", arg3: 0, args: null }, drop: { typ: "", id: 0, num: 0, args: null, wgt: 0, lv: 0 }, troop_composition: [ { typ: "hero", id: 19203014, lv: 2, val: 100, skills: [ 1924505301 ] }, { typ: "hero", id: 19203011, lv: 2, val: 100, skills: [ 1924504101 ] } ], attached_troop: [], lifetime: 18000000, lc_name: "NPC_ground_denfence", soldier_category: 11221006 }, npcowner: 0, fieldtype: 0, wildbuildingid: 0, embattletroopid: 0, cultivatetroopid: 0, farmnpctroopid: 0 }, multi: false, upsert: true } planSummary: IDHACK keysExamined:1 docsExamined:1 nMatched:1 nModified:1 numYields:1 locks:{ Global: { acquireCount: { r: 2, w: 2 } }, Database: { acquireCount: { w: 2 } }, Collection: { acquireCount: { w: 2 } } } 114ms
2021-12-30T11:36:00.132+0800 I CONTROL [main] ***** SERVER RESTARTED *****
2021-12-30T11:36:00.177+0800 I CONTROL [initandlisten] MongoDB starting : pid=605129 port=27017 dbpath=/var/lib/mongodb 64-bit host=ubuntu-dev
2021-12-30T11:36:00.177+0800 I CONTROL [initandlisten] db version v3.6.8
没发现什么错误
- 查看系统日志
vi /var/log/syslog
Dec 30 11:11:52 ubuntu-dev systemd[3853]: snap.go.go.45773cd3-a6e7-4f23-966e-b8e634a7f5e3.scope: Succeeded.
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354218] game invoked oom-killer: gfp_mask=0x100cca(GFP_HIGHUSER_MOVABLE), order=0, oom_score_adj=0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354225] CPU: 6 PID: 604773 Comm: game Not tainted 5.11.0-40-generic #44~20.04.2-Ubuntu
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354228] Hardware name: VMware, Inc. VMware7,1/440BX Desktop Reference Platform, BIOS VMW71.00V.7581552.B64.1801142334 01/14/2018
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354230] Call Trace:
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354233] dump_stack+0x74/0x92
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354245] dump_header+0x4f/0x1f6
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354250] oom_kill_process.cold+0xb/0x10
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354252] out_of_memory.part.0+0x1ee/0x460
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354258] out_of_memory+0x6d/0xd0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354260] __alloc_pages_slowpath.constprop.0+0xc4d/0xd20
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354268] __alloc_pages_nodemask+0x2a0/0x300
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354271] alloc_pages_current+0x87/0xe0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354277] __page_cache_alloc+0x89/0xb0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354282] pagecache_get_page+0xf1/0x350
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354284] filemap_fault+0x9f3/0xfc0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354286] ? update_load_avg+0x7c/0x5e0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354293] ? filemap_map_pages+0x218/0x3f0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354294] ext4_filemap_fault+0x32/0x50
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354301] __do_fault+0x3c/0xe0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354307] do_fault+0x276/0x4f0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354310] __handle_mm_fault+0x677/0x920
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354312] handle_mm_fault+0xd7/0x2b0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354313] do_user_addr_fault+0x1a0/0x450
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354320] exc_page_fault+0x69/0x150
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354325] ? asm_exc_page_fault+0x8/0x30
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354331] asm_exc_page_fault+0x1e/0x30
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354334] RIP: 0033:0x46bfdd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354339] Code: Unable to access opcode bytes at RIP 0x46bfb3.
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354340] RSP: 002b:00007f68f2947db8 EFLAGS: 00010202
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354343] RAX: 0000000000000000 RBX: 0000000000002710 RCX: 000000000046bfdd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354344] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 00007f68f2947db8
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354345] RBP: 00007f68f2947dc8 R08: 0000000000002e21 R09: 00000000003df47e
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354347] R10: 0000000000000000 R11: 0000000000000202 R12: 00007f68f29477f8
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354348] R13: 00007ffe2b381dcf R14: 000000c000000b60 R15: 00007f68f2947fc0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354350] Mem-Info:
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354352] active_anon:1450335 inactive_anon:469416 isolated_anon:0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354352] active_file:108 inactive_file:22 isolated_file:0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354352] unevictable:0 dirty:0 writeback:0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354352] slab_reclaimable:17473 slab_unreclaimable:24422
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354352] mapped:197 shmem:129 pagetables:8484 bounce:0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354352] free:25354 free_pcp:2452 free_cma:0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354356] Node 0 active_anon:5801340kB inactive_anon:1877664kB active_file:432kB inactive_file:88kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:788kB dirty:0kB writeback:0kB shmem:516kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 471040kB writeback_tmp:0kB kernel_stack:14288kB pagetables:33936kB all_unreclaimable? no
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354361] Node 0 DMA free:14868kB min:132kB low:164kB high:196kB reserved_highatomic:0KB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15996kB managed:15908kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354366] lowmem_reserve[]: 0 2895 7873 7873 7873
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354370] Node 0 DMA32 free:44380kB min:24808kB low:31008kB high:37208kB reserved_highatomic:0KB active_anon:2329532kB inactive_anon:598400kB active_file:0kB inactive_file:520kB unevictable:0kB writepending:0kB present:3128688kB managed:3030804kB mlocked:0kB bounce:0kB free_pcp:5336kB local_pcp:560kB free_cma:0kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354374] lowmem_reserve[]: 0 0 4977 4977 4977
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354377] Node 0 Normal free:42168kB min:42640kB low:53300kB high:63960kB reserved_highatomic:0KB active_anon:3471808kB inactive_anon:1279036kB active_file:176kB inactive_file:636kB unevictable:0kB writepending:0kB present:5242880kB managed:5097120kB mlocked:0kB bounce:0kB free_pcp:4472kB local_pcp:508kB free_cma:0kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354382] lowmem_reserve[]: 0 0 0 0 0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354385] Node 0 DMA: 1*4kB (U) 0*8kB 1*16kB (U) 0*32kB 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 0*1024kB 1*2048kB (M) 3*4096kB (M) = 14868kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354396] Node 0 DMA32: 684*4kB (UME) 747*8kB (UME) 524*16kB (UME) 289*32kB (UME) 178*64kB (UME) 38*128kB (UME) 2*256kB (ME) 0*512kB 1*1024kB (U) 0*2048kB 0*4096kB = 44136kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354408] Node 0 Normal: 2197*4kB (UME) 1792*8kB (UME) 681*16kB (UME) 129*32kB (UME) 34*64kB (UME) 10*128kB (ME) 2*256kB (M) 0*512kB 0*1024kB 0*2048kB 0*4096kB = 42116kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354421] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354422] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354424] 12740 total pagecache pages
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354426] 12204 pages in swap cache
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354427] Swap cache stats: add 7328025, delete 7316083, find 5789447/7076218
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354428] Free swap = 0kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354429] Total swap = 2097148kB
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354430] 2096891 pages RAM
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354430] 0 pages HighMem/MovableOnly
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354431] 60933 pages reserved
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354432] 0 pages hwpoisoned
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354433] Tasks state (memory values in pages):
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354433] [ pid ] uid tgid total_vm rss pgtables_bytes swapents oom_score_adj name
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354452] [ 417] 0 417 17074 153 106496 316 -250 systemd-journal
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354456] [ 465] 0 465 21283 0 49152 68 0 vmware-vmblock-
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354460] [ 488] 0 488 6136 15 65536 998 -1000 systemd-udevd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354463] [ 503] 100 503 4604 46 77824 183 0 systemd-network
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354465] [ 792] 101 792 6009 64 86016 984 0 systemd-resolve
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354467] [ 793] 102 793 22565 27 81920 184 0 systemd-timesyn
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354469] [ 797] 0 797 13373 0 86016 374 0 VGAuthService
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354471] [ 799] 0 799 60993 197 98304 170 0 vmtoolsd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354473] [ 820] 0 820 637 0 45056 42 0 acpid
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354475] [ 823] 115 823 2275 172 53248 91 0 avahi-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354478] [ 824] 0 824 3124 26 53248 39 0 cron
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354480] [ 826] 103 826 2256 342 53248 202 -900 dbus-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354482] [ 832] 0 832 20498 44 57344 59 0 irqbalance
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354484] [ 833] 0 833 10601 1 110592 2064 0 networkd-dispat
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354487] [ 834] 0 834 58785 242 94208 527 0 polkitd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354489] [ 836] 104 836 56089 112 90112 460 0 rsyslogd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354491] [ 853] 0 853 59725 1 86016 198 0 switcheroo-cont
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354493] [ 854] 0 854 4490 75 77824 487 0 systemd-logind
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354495] [ 856] 0 856 98205 148 131072 421 0 udisksd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354497] [ 857] 0 857 3422 14 65536 145 0 wpa_supplicant
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354499] [ 864] 115 864 2088 1 49152 81 0 avahi-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354501] [ 932] 121 932 62386 126 110592 1186 0 colord
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354503] [ 934] 129 934 3058942 3272 389120 2108 0 etcd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354506] [ 944] 0 944 30298 0 131072 2066 0 unattended-upgr
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354508] [ 945] 120 945 63275 3 131072 471 0 whoopsie
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354510] [ 949] 116 949 2816 25 65536 94 0 kerneloops
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354512] [ 951] 116 951 2816 24 65536 94 0 kerneloops
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354514] [ 952] 0 952 60005 68 106496 366 0 ModemManager
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354516] [ 956] 0 956 60678 0 90112 275 0 gdm3
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354518] [ 965] 127 965 16273 529 110592 1280 0 redis-server
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354523] [ 979] 0 979 3046 0 69632 245 -1000 sshd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354525] [ 1064] 0 1064 42505 0 94208 301 0 gdm-session-wor
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354528] [ 1139] 125 1139 4704 9 77824 474 0 systemd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354530] [ 1141] 0 1141 63834 74 110592 265 0 upowerd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354532] [ 1143] 125 1143 42226 1 98304 865 0 (sd-pam)
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354535] [ 1163] 125 1163 202007 0 147456 523 0 pulseaudio
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354537] [ 1165] 125 1165 128714 1 184320 2132 0 tracker-miner-f
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354539] [ 1169] 125 1169 41219 1 77824 146 0 gdm-wayland-ses
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354541] [ 1171] 125 1171 1850 0 53248 171 0 dbus-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354543] [ 1193] 111 1193 38235 0 69632 63 0 rtkit-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354545] [ 1195] 125 1195 1325 0 45056 48 0 dbus-run-sessio
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354547] [ 1196] 125 1196 1914 1 53248 237 0 dbus-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354549] [ 1197] 125 1197 103388 0 151552 601 0 gnome-session-b
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354551] [ 1208] 125 1208 60742 0 98304 252 0 gvfsd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354553] [ 1244] 125 1244 78202 0 94208 222 0 gvfsd-fuse
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354555] [ 1252] 125 1252 79320 0 110592 465 0 gvfs-udisks2-vo
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354557] [ 1260] 125 1260 79994 0 114688 299 0 gvfs-afc-volume
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354559] [ 1266] 125 1266 60300 0 90112 250 0 gvfs-gphoto2-vo
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354561] [ 1275] 125 1275 59731 0 90112 211 0 gvfs-mtp-volume
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354562] [ 1280] 125 1280 59782 0 90112 203 0 gvfs-goa-volume
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354564] [ 1285] 125 1285 137598 0 258048 1544 0 goa-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354566] [ 1306] 125 1306 1415618 5767 1126400 15171 0 gnome-shell
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354568] [ 1316] 125 1316 79531 0 110592 294 0 goa-identity-se
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354571] [ 1382] 125 1382 76337 0 86016 188 0 at-spi-bus-laun
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354573] [ 1387] 125 1387 1779 0 53248 109 0 dbus-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354575] [ 1400] 125 1400 402820 0 450560 2691 0 Xwayland
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354577] [ 1464] 125 1464 59737 1 94208 156 0 xdg-permission-
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354579] [ 1474] 125 1474 733386 1 266240 1310 0 gjs
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354581] [ 1476] 125 1476 40693 0 77824 191 0 at-spi2-registr
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354583] [ 1486] 125 1486 117004 0 139264 332 0 gsd-sharing
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354585] [ 1489] 125 1489 85783 0 159744 1221 0 gsd-wacom
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354587] [ 1490] 125 1490 141500 131 188416 1161 0 gsd-color
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354589] [ 1491] 125 1491 85841 0 163840 1227 0 gsd-keyboard
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354591] [ 1492] 125 1492 62912 58 118784 323 0 gsd-print-notif
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354594] [ 1493] 125 1493 115093 0 114688 206 0 gsd-rfkill
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354595] [ 1494] 125 1494 79633 1 114688 270 0 gsd-smartcard
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354597] [ 1495] 125 1495 94248 0 163840 521 0 gsd-datetime
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354599] [ 1496] 125 1496 189937 0 196608 1380 0 gsd-media-keys
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354601] [ 1497] 125 1497 59695 0 81920 162 0 gsd-screensaver
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354602] [ 1498] 125 1498 80683 0 118784 295 0 gsd-sound
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354604] [ 1499] 125 1499 78257 0 90112 185 0 gsd-a11y-settin
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354606] [ 1500] 125 1500 78774 73 110592 143 0 gsd-housekeepin
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354608] [ 1502] 125 1502 104515 0 172032 1328 0 gsd-power
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354611] [ 1555] 125 1555 86365 1 155648 497 0 gsd-printer
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354612] [ 1618] 125 1618 97007 1 110592 373 0 ibus-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354615] [ 1623] 125 1623 41508 0 81920 170 0 ibus-memconf
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354617] [ 1625] 125 1625 445628 0 503808 2349 0 ibus-x11
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354619] [ 1634] 125 1634 59958 0 98304 179 0 ibus-portal
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354621] [ 1641] 125 1641 41506 0 90112 184 0 ibus-engine-sim
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354623] [ 3853] 1000 3853 4709 196 81920 292 0 systemd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354625] [ 3854] 1000 3854 42548 1 102400 1196 0 (sd-pam)
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354627] [ 3862] 1000 3862 165581 475 196608 1672 0 tracker-miner-f
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354628] [ 3868] 1000 3868 1853 101 49152 69 0 dbus-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354631] [ 3900] 1000 3900 60741 0 90112 272 0 gvfsd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354633] [ 3905] 1000 3905 78202 0 94208 224 0 gvfsd-fuse
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354635] [ 3911] 1000 3911 79321 0 110592 491 0 gvfs-udisks2-vo
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354637] [ 3922] 1000 3922 79994 0 106496 313 0 gvfs-afc-volume
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354639] [ 3927] 1000 3927 60300 0 94208 234 0 gvfs-gphoto2-vo
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354641] [ 3932] 1000 3932 59731 0 86016 208 0 gvfs-mtp-volume
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354642] [ 3936] 1000 3936 59782 0 86016 198 0 gvfs-goa-volume
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354644] [ 3940] 1000 3940 137631 0 262144 1542 0 goa-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354647] [ 3953] 1000 3953 79531 0 106496 293 0 goa-identity-se
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354649] [ 4077] 1000 4077 5332 1205 69632 1047 0 tmux: server
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354650] [ 4078] 1000 4078 3837 2 61440 711 0 bash
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354652] [ 6038] 1000 6038 3559 0 57344 473 0 bash
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354654] [ 6082] 1000 6082 3492 0 53248 460 0 bash
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354656] [ 6136] 1000 6136 3608 2 61440 493 0 bash
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354658] [ 6178] 1000 6178 3843 2 65536 736 0 bash
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354660] [ 537646] 0 537646 60566 165 98304 266 0 accounts-daemon
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354662] [ 538159] 125 538159 41376 0 81920 191 0 gvfsd-metadata
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354665] [ 538217] 1000 538217 41376 0 81920 182 0 gvfsd-metadata
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354668] [2484966] 1000 2484966 109505 442 188416 2673 0 tracker-store
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354671] [2815808] 0 2815808 402999 1780 344064 2852 -900 snapd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354673] [3662647] 0 3662647 18731 71 126976 517 0 nmbd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354675] [3662657] 0 3662657 22584 56 163840 708 0 smbd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354699] [3662659] 0 3662659 22037 21 147456 711 0 smbd-notifyd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354701] [3662660] 0 3662660 22039 37 143360 695 0 cleanupd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354703] [3662662] 0 3662662 22580 150 159744 634 0 lpqd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354705] [ 235963] 1000 235963 626061 87480 1523712 41926 0 game
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354708] [ 259557] 0 259557 3497 27 69632 341 0 sshd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354710] [ 259643] 1000 259643 3530 34 69632 338 0 sshd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354712] [ 259644] 1000 259644 3491 2 57344 427 0 bash
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354714] [ 579691] 0 579691 3497 16 61440 353 0 sshd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354716] [ 579774] 1000 579774 3530 47 61440 331 0 sshd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354718] [ 579775] 1000 579775 3860 1 65536 713 0 bash
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354720] [ 587465] 1000 587465 42730 0 131072 135 0 git
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354723] [ 587466] 1000 587466 2880 0 49152 78 0 pager
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354784] [ 588478] 1000 588478 497875 7137 704512 3170 0 gate
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354786] [ 592624] 1000 592624 42759 1 143360 137 0 git
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354788] [ 592625] 1000 592625 2880 0 53248 78 0 pager
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354790] [ 592954] 128 592954 1345268 734560 9166848 315485 0 mongod
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354793] [ 600410] 0 600410 7934 153 90112 293 0 cupsd
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354795] [ 600412] 0 600412 44759 172 118784 332 0 cups-browsed
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354797] [ 600420] 7 600420 3834 81 69632 119 0 dbus
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354800] [ 602369] 1000 602369 2882 0 53248 79 0 make
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354802] [ 602370] 1000 602370 493585 1 303104 4227 0 go
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354804] [ 602476] 1000 602476 353896 119872 1585152 61715 0 compile
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354807] [ 604772] 1000 604772 1399595 939455 8060928 16557 0 game
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.354809] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/system.slice/mongodb.service,task=mongod,pid=592954,uid=128
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.355054] Out of memory: Killed process 592954 (mongod) total-vm:5381072kB, anon-rss:2938240kB, file-rss:0kB, shmem-rss:0kB, UID:128 pgtables:8952kB oom_score_adj:0
Dec 30 11:16:19 ubuntu-dev kernel: [4060375.701894] oom_reaper: reaped process 592954 (mongod), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB
Dec 30 11:16:20 ubuntu-dev systemd[1]: mongodb.service: Main process exited, code=killed, status=9/KILL
Dec 30 11:16:20 ubuntu-dev systemd[1]: mongodb.service: Failed with result 'signal'.
Dec 30 11:17:01 ubuntu-dev CRON[604802]: (root) CMD ( cd / && run-parts --report /etc/cron.hourly)
关闭oom killer 最近有位 VPS 客户抱怨 MySQL 无缘无故挂掉，还有位客户抱怨 VPS 经常死机，登陆到终端看了一下，都是常见的 Out of memory 问题。这通常是因为某时刻应用程序大量请求内存导致系统内存不足造成的，这通常会触发 Linux内核里的 Out of Memory (OOM) killer，OOM killer 会杀掉某个进程以腾出内存留给系统用，不致于让系统立刻崩溃。如果检查相关的日志文件（/var/log/messages）就会看到下面类似的 Out of memory: Kill process 信息：
...
Out of memory: Kill process 9682 (mysqld) score 9 or sacrifice child
Killed process 9682, UID 27, (mysqld) total-vm:47388kB, anon-rss:3744kB, file-rss:80kB
httpd invoked oom-killer: gfp_mask=0x201da, order=0, oom_adj=0, oom_score_adj=0
httpd cpuset=/ mems_allowed=0
Pid: 8911, comm: httpd Not tainted 2.6.32-279.1.1.el6.i686 #1
...
21556 total pagecache pages
21049 pages in swap cache
Swap cache stats: add 12819103, delete 12798054, find 3188096/4634617
Free swap = 0kB
Total swap = 524280kB
131071 pages RAM
0 pages HighMem
3673 pages reserved
67960 pages shared
124940 pages non-shared
Linux 内核根据应用程序的要求分配内存，通常来说应用程序分配了内存但是并没有实际全部使用，为了提高性能，这部分没用的内存可以留作它用，这部分内存是属于每 个进程的，内核直接回收利用的话比较麻烦，所以内核采用一种过度分配内存（over-commit memory）的办法来间接利用这部分 “空闲” 的内存，提高整体内存的使用效率。一般来说这样做没有问题，但当大多数应用程序都消耗完自己的内存的时候麻烦就来了，因为这些应用程序的内存需求加起来超 出了物理内存（包括 swap）的容量，内核（OOM killer）必须杀掉一些进程才能腾出空间保障系统正常运行。用银行的例子来讲可能更容易懂一些，部分人取钱的时候银行不怕，银行有足够的存款应付，当 全国人民（或者绝大多数）都取钱而且每个人都想把自己钱取完的时候银行的麻烦就来了，银行实际上是没有这么多钱给大家取的。
内核检测到系统内存不足、挑选并杀掉某个进程的过程可以参考内核源代码 linux/mm/oom_kill.c， 当系统内存不足的时候，out_of_memory() 被触发，然后调用 select_bad_process() 选择一个 “bad” 进程杀掉，如何判断和选择一个 “bad” 进程呢，总不能随机选吧？挑选的过程由 oom_badness() 决定，挑选的算法和想法都很简单很朴实：最 bad 的那个进程就是那个最占用内存的进程。
/**
* oom_badness - heuristic function to determine which candidate task to kill
* @p: task struct of which task we should calculate
* @totalpages: total present RAM allowed for page allocation
*
* The heuristic for determining which task to kill is made to be as simple and
* predictable as possible. The goal is to return the highest value for the
* task consuming the most memory to avoid subsequent oom failures.
*/
unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
const nodemask_t *nodemask, unsigned long totalpages)
{
long points;
long adj;
if (oom_unkillable_task(p, memcg, nodemask))
return 0;
p = find_lock_task_mm(p);
if (!p)
return 0;
adj = (long)p->signal->oom_score_adj;
if (adj == OOM_SCORE_ADJ_MIN) {
task_unlock(p);
return 0;
}
/*
* The baseline for the badness score is the proportion of RAM that each
* task's rss, pagetable and swap space use.
*/
points = get_mm_rss(p->mm) + p->mm->nr_ptes +
get_mm_counter(p->mm, MM_SWAPENTS);
task_unlock(p);
/*
* Root processes get 3% bonus, just like the __vm_enough_memory()
* implementation used by LSMs.
*/
if (has_capability_noaudit(p, CAP_SYS_ADMIN))
adj -= 30;
/* Normalize to oom_score_adj units */
adj *= totalpages / 1000;
points += adj;
/*
* Never return 0 for an eligible task regardless of the root bonus and
* oom_score_adj (oom_score_adj can't be OOM_SCORE_ADJ_MIN here).
*/
return points > 0 ? points : 1;
}
理解了这个算法我们就理解了为啥 MySQL 躺着也能中枪了，因为它的体积总是最大（一般来说它在系统上占用内存最多），所以如果 Out of Memeory (OOM) 的话总是不幸第一个被 kill 掉。解决这个问题最简单的办法就是增加内存，或者想办法优化 MySQL 使其占用更少的内存，除了优化 MySQL 外还可以优化系统，让系统尽可能使用少的内存以便应用程序（如 MySQL) 能使用更多的内存，还有一个临时的办法就是调整内核参数，让 MySQL 进程不容易被 OOM killer 发现。
配置 OOM killer 我们可以通过一些内核参数来调整 OOM killer 的行为，避免系统在那里不停的杀进程。比如我们可以在触发 OOM 后立刻触发 kernel panic，kernel panic 10秒后自动重启系统。
# sysctl -w vm.panic_on_oom=1
vm.panic_on_oom = 1
# sysctl -w kernel.panic=10
kernel.panic = 10
# echo "vm.panic_on_oom=1" >> /etc/sysctl.conf
# echo "kernel.panic=10" >> /etc/sysctl.conf
从上面的 oom_kill.c 代码里可以看到 oom_badness() 给每个进程打分，根据 points 的高低来决定杀哪个进程，这个 points 可以根据 adj 调节，root 权限的进程通常被认为很重要，不应该被轻易杀掉，所以打分的时候可以得到 3% 的优惠（adj -= 30; 分数越低越不容易被杀掉）。我们可以在用户空间通过操作每个进程的 oom_adj 内核参数来决定哪些进程不这么容易被 OOM killer 选中杀掉。比如，如果不想 MySQL 进程被轻易杀掉的话可以找到 MySQL 运行的进程号后，调整 oom_score_adj 为 -15（注意 points 越小越不容易被杀）：
# ps aux | grep mysqld
mysql 2196 1.6 2.1 623800 44876 ? Ssl 09:42 0:00 /usr/sbin/mysqld
# cat /proc/2196/oom_score_adj
0
# echo -15 > /proc/2196/oom_score_adj
当然，如果需要的话可以完全关闭 OOM killer（不推荐用在生产环境）：
# sysctl -w vm.overcommit_memory=2
# echo "vm.overcommit_memory=2" >>