求助。。。。关于内核缓存申请coredump

code_digger 2017-04-19 08:01:00

在嵌入式板子网口上，板子上跑rtspserver并将视频流发给PC,PC使用VLC来播放，晚上拷机，在正常运行一个晚上后，VLC视频断流，嵌入式板端出现coredump(挂在红色字体部分),LOG如下：
/ # ?[73133.931334] ------------[ cut here ]------------
[73133.935960] kernel BUG at /linux/mm/slab.c:3005!
[73133.944977] Internal error: Oops - BUG: 0 [#1] PREEMPT ARM
[73133.950439] Modules linked in: cdc_acm qcserial option usb_wwan usbserial rndis_host cdc_ether qmi_wwan cdc_wdm usbnet ohci_hcd ehci_hcd usbcore usb_common cfg80211
[73133.965699] CPU: 0 PID: 566 Comm: RTSPServer Not tainted 3.10.71 #1
[73133.972291] task: 4331a700 ti: 42bae000 task.ti: 42bae000
[73133.977680] PC is at cache_alloc_refill+0x3b8/0xcf8
[73133.982541] LR is at 0x1d
[73133.985145] pc : [<400c46d8>] lr : [<0000001d>] psr: 600f0013
[73133.985145] sp : 42bafb68 ip : 00000020 fp : 432683c0
[73133.996589] r10: 00100100 r9 : 00200200 r8 : 4326a800
[73134.001790] r7 : 4057a480 r6 : 0000001f r5 : 43334000 r4 : 432077c0
[73134.008290] r3 : 42cf3000 r2 : 00000014 r1 : 00000014 r0 : 00000014
[73134.014795] Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment user
[73134.021904] Control: 10c5387d Table: 1f230059 DAC: 00000015
[73134.027626] Process RTSPServer (pid: 566, stack limit = 0x42bae238)
[73134.034214] Stack: (0x42bafb68 to 0x42bb0000)
[73134.038553] fb60: 00000048 000004d0 00000000 40534bc0 00000001 40334d70
[73134.046705] fb80: 00000001 00000002 05ad104c 000004d0 000005e3 432077c0 40534bc0 200f0013
[73134.054856] fba0: 00000000 000004d0 00000000 400c50fc 4350a040 000005e3 000004d0 432077c0
[73134.063006] fbc0: 00000000 402fc6cc 00000000 00000000 00000000 4350a040 00000000 42bae000
[73134.071156] fbe0: 00000000 00000000 00000000 402f6a98 433fd3cc 000005e3 00000000 720a7f64
[73134.079306] fc00: 00000000 42cf3980 00000045 00000004 00000000 00000000 00000000 000005b0
[73134.087456] fc20: 000005c4 4331a998 00000000 435ae7c0 00000000 402f6d88 42bafc8c 4032c9c8
[73134.095607] fc40: 403424ac 403430dc 00000000 4052fa60 006f355b 000005dc 4350a040 000005dc
[73134.103758] fc60: 000005b0 42bafcdc 00000014 00000000 00000010 000005c8 00000003 42bafcd0
[73134.111909] fc80: 000005c8 000005b0 0000000e 4052fa60 012aa8c0 4331a700 4350a040 4350a1f8
[73134.120060] fca0: 42bafcd0 403429d4 42baff08 00000000 42bafd58 40344810 4331a998 403429d4
[73134.128210] fcc0: 42baff08 000005b0 00000008 00000040 42bafcd0 42bafcd0 00000000 00000000
[73134.136361] fce0: 00000000 00000000 000005dc 000005b0 435ae7c0 4350a000 0000002c 4350a040
[73134.144511] fd00: 4350a1f8 42bafe18 4052f008 000005a8 000005b0 4036a0ac 000005b0 00000008
[73134.152661] fd20: 42bafd5c 42bafd58 00000040 0000000c 4340a840 4350a1f8 00000000 403429d4
[73134.160813] fd40: 42bafd5c 000064de 032aa8c0 00000000 012aa8c0 43524780 00000000 032aa8c0
[73134.168963] fd60: 00000000 00000000 00000000 00000008 00000100 40325ff8 432077c0 400c3c14
[73134.177113] fd80: 42d02180 43524780 00000000 40534bc0 00000100 403079ac 00000001 40572688
[73134.185264] fda0: 00000000 4057268c 40572640 4002cb34 00000001 00000084 0000001b 00000000
[73134.193414] fdc0: 405370e8 40534bc0 40572640 720a7f64 006f34fc 00000000 40373220 00000040
[73134.201564] fde0: 42e90c80 42bafe18 00000002 42bafdf8 00000001 402f3a2c 00000002 42e90ca0
[73134.209715] fe00: 40546170 000005a8 42e90c80 4010d2d8 00000000 42bafe18 00000000 00000000
[73134.217866] fe20: 42baff08 00000002 00000000 00000000 00000040 42bafe68 00000001 42baff80
[73134.226017] fe40: 000005a8 00000000 402f3850 4331a700 43330d40 42baff08 00000002 400c7bd4
[73134.234166] fe60: 00000000 00000000 00000001 43330d40 00000000 00000000 00000000 4331a700
[73134.242316] fe80: 00000000 00000000 00000000 00000000 42bafdf8 00000000 000005a8 00000000
[73134.250466] fea0: 000005a8 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[73134.258617] fec0: 00000000 00000000 00000010 43330d40 402f3850 00000001 42baff08 00000002
[73134.266767] fee0: 000005a8 00000000 0000c000 400c8d64 42baff80 402f3850 00000000 00000000
[73134.274918] ff00: 00000000 42baff08 0182cc14 0000000c 26fdf531 0000059c 43330688 00000001
[73134.283067] ff20: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000001
[73134.291218] ff40: 43330688 00000000 42e59d08 43330d40 3e898b98 00000000 00000000 00000001
[73134.299368] ff60: 00000002 00000000 0182cc82 400c9124 42baff80 00000092 4000f414 400c9204
[73134.307519] ff80: 00000000 00000000 3e898c80 00000002 3e898b98 0000000d 00000092 4000f414
[73134.315669] ffa0: 42bae000 4000f1c0 00000002 3e898b98 0000000d 3e898b98 00000002 00000000
[73134.323819] ffc0: 00000002 3e898b98 0000000d 00000092 000339b0 0182cc24 be9f6bf4 0182cc82
[73134.331969] ffe0: 00000092 3e898b68 36da0cf9 36d275b6 800f0030 0000000d 008fd801 008fdc01
[73134.340140] [<400c46d8>] (cache_alloc_refill+0x3b8/0xcf8) from [<400c50fc>] (kmem_cache_alloc+0xe4/0x120)
[73134.349695] [<400c50fc>] (kmem_cache_alloc+0xe4/0x120) from [<402fc6cc>] (_alloc_skb+0x44/0x130)
[73134.358548] [<402fc6cc>] (_alloc_skb+0x44/0x130) from [<402f6a98>] (sock_alloc_send_pskb+0x2dc/0x5b4)
[73134.367831] [<402f6a98>] (sock_alloc_send_pskb+0x2dc/0x5b4) from [<402f6d88>] (sock_alloc_send_skb+0x18/0x20)
[73134.377726] [<402f6d88>] (sock_alloc_send_skb+0x18/0x20) from [<403430dc>] (_ip_append_data.isra.44+0x660/0x984)
[73134.387963] [<403430dc>] (_ip_append_data.isra.44+0x660/0x984) from [<40344810>] (ip_make_skb+0xac/0xe8)
[73134.397513] [<40344810>] (ip_make_skb+0xac/0xe8) from [<4036a0ac>] (udp_sendmsg+0x220/0x7c4)
[73134.405929] [<4036a0ac>] (udp_sendmsg+0x220/0x7c4) from [<402f3a2c>] (sock_aio_write+0x1dc/0x208)
[73134.414786] [<402f3a2c>] (sock_aio_write+0x1dc/0x208) from [<400c7bd4>] (do_sync_readv_writev+0x74/0x9c)
[73134.424245] [<400c7bd4>] (do_sync_readv_writev+0x74/0x9c) from [<400c8d64>] (do_readv_writev+0xbc/0x3c0)
[73134.433701] [<400c8d64>] (do_readv_writev+0xbc/0x3c0) from [<400c9124>] (vfs_writev+0x48/0x74)
[73134.442289] [<400c9124>] (vfs_writev+0x48/0x74) from [<400c9204>] (SyS_writev+0x3c/0x78)
[73134.450365] [<400c9204>] (SyS_writev+0x3c/0x78) from [<4000f1c0>] (ret_fast_syscall+0x0/0x90)
[73134.458867] Code: e5950010 e5942018 e1500002 3affff74 (e7f001f2)
[73134.464953] ---[ end trace fee194a0c24fae4e ]---

linux/mm/slab.c:3005!
其对应的函数为：
static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags, bool force_refill)
{
int batchcount;
struct kmem_cache_node *n;
struct array_cache *ac;
int node;

check_irq_off();
node = numa_mem_id();
if (unlikely(force_refill))
goto force_grow;
retry:
ac = cpu_cache_get(cachep);
batchcount = ac->batchcount;
if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
/*
* If there was little recent activity on this cache, then
* perform only a partial refill. Otherwise we could generate
* refill bouncing.
*/
batchcount = BATCHREFILL_LIMIT;
}
n = cachep->node[node];

BUG_ON(ac->avail > 0 || !n);
spin_lock(&n->list_lock);

/* See if we can refill from the shared array */
if (n->shared && transfer_objects(ac, n->shared, batchcount)) {
n->shared->touched = 1;
goto alloc_done;
}

while (batchcount > 0) {
struct list_head *entry;
struct slab *slabp;
/* Get slab alloc is to come from. */
entry = n->slabs_partial.next;
if (entry == &n->slabs_partial) {
n->free_touched = 1;
entry = n->slabs_free.next;
if (entry == &n->slabs_free)
goto must_grow;
}

slabp = list_entry(entry, struct slab, list);
check_slabp(cachep, slabp);
check_spinlock_acquired(cachep);

/*
* The slab was either on partial or free list so
* there must be at least one object available for
* allocation.
*/
BUG_ON(slabp->inuse >= cachep->num);

while (slabp->inuse < cachep->num && batchcount--) {
STATS_INC_ALLOCED(cachep);
STATS_INC_ACTIVE(cachep);
STATS_SET_HIGH(cachep);

ac_put_obj(cachep, ac, slab_get_obj(cachep, slabp,
node));
}
check_slabp(cachep, slabp);

/* move slabp to correct slabp list: */
list_del(&slabp->list);
if (slabp->free == BUFCTL_END)
list_add(&slabp->list, &n->slabs_full);
else
list_add(&slabp->list, &n->slabs_partial);
}

must_grow:
n->free_objects -= ac->avail;
alloc_done:
spin_unlock(&n->list_lock);

if (unlikely(!ac->avail)) {
int x;
force_grow:
x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL);

/* cache_grow can reenable interrupts, then ac could change. */
ac = cpu_cache_get(cachep);
node = numa_mem_id();

/* no objects in sight? abort */
if (!x && (ac->avail == 0 || force_refill))
return NULL;

if (!ac->avail) /* objects refilled by interrupt? */
goto retry;
}
ac->touched = 1;

return ac_get_obj(cachep, ac, flags, force_refill);
}
3005 行其对应地方为：
BUG_ON(slabp->inuse >= cachep->num);
根据函数的逻辑，代码走到这里的时候，说明刚从cache的partial或者free链表里取得了一个可用slab，而这个获取的slab，一定是还有可用的obj，即slabp->inuse一定小于上限cachep->num。此处出错，是否说明内核缓存无法申请导致？
由于发送视频数据包涉及频繁的申请释放内存操作，是否释放比申请慢导致？在内核内存管理的那些 code可以做跟踪确认？
考虑到slab本身的错误这种情况的可能性很低，有哪些可能的方向导致上面的coredump?

...全文