BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 169s!
BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 168s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=9/256 refcnt=10
in-flight: 5953:ovs_dp_masks_rebalance
pending: ovs_dp_masks_rebalance, 2*nsim_dev_hwstats_traffic_work, 2*psi_avgs_work, kfree_rcu_monitor, 2*ovs_dp_masks_rebalance
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=11/256 refcnt=12
in-flight: 8:free_obj_work
pending: 2*nsim_dev_hwstats_traffic_work, ovs_dp_masks_rebalance, vmstat_shepherd, psi_avgs_work, ovs_dp_masks_rebalance, 2*psi_avgs_work, kfree_rcu_monitor, ima_keys_handler
workqueue events_long: flags=0x0
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: 2*defense_work_handler
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=3/256 refcnt=4
pending: 3*defense_work_handler
workqueue events_unbound: flags=0x2
pwq 4: cpus=0-1 flags=0x4 nice=0 active=8/512 refcnt=9
in-flight: 7337:cfg80211_wiphy_work cfg80211_wiphy_work, 34:cfg80211_wiphy_work
pending: cfg80211_wiphy_work, 2*nsim_dev_trap_report_work, flush_memcg_stats_dwork, macvlan_process_broadcast
pwq 4: cpus=0-1 flags=0x4 nice=0 active=23/512 refcnt=24
in-flight: 6271:cfg80211_wiphy_work, 50:cfg80211_wiphy_work cfg80211_wiphy_work, 6279:nsim_dev_trap_report_work, 11:cfg80211_wiphy_work, 1085:cfg80211_wiphy_work cfg80211_wiphy_work, 49:nsim_dev_trap_report_work, 12:cfg80211_wiphy_work
pending: toggle_allocation_gate, nsim_dev_trap_report_work, 4*cfg80211_wiphy_work, 5*macvlan_process_broadcast, crng_reseed, 2*macvlan_process_broadcast
workqueue events_freezable: flags=0x4
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x80
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=7/256 refcnt=8
pending: wg_ratelimiter_gc_entries, neigh_managed_work, neigh_periodic_work, neigh_managed_work, neigh_periodic_work, gc_worker, do_cache_clean
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=2/256 refcnt=3
pending: reg_check_chans_work, check_lifetime
workqueue mm_percpu_wq: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: vmstat_update
workqueue writeback: flags=0x4a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wb_workfn
workqueue dm_bufio_cache: flags=0x8
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: work_fn
workqueue ipv6_addrconf: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=8
pending: addrconf_verify_work
inactive: 4*addrconf_verify_work
workqueue krxrpcd: flags=0xa001a
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=8
in-flight: 5867:rxrpc_peer_keepalive_worker
inactive: 4*rxrpc_peer_keepalive_worker
workqueue bat_events: flags=0xe000a
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/1 refcnt=39
in-flight: 2923:batadv_iv_send_outstanding_bat_ogm_packet
inactive: 4*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_mcast_mla_update, 4*batadv_nc_worker, 3*batadv_iv_send_outstanding_bat_ogm_packet, 4*batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 2*batadv_tt_purge
workqueue hci0: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
pending: hci_conn_timeout
workqueue hci1: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
pending: hci_conn_timeout
workqueue hci2: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
pending: hci_conn_timeout
workqueue hci3: flags=0xa0012
pwq 5: cpus=0-1 node=0 flags=0x4 nice=-20 active=1/1 refcnt=4
pending: hci_conn_timeout
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x6
pwq 4: cpus=0-1 flags=0x4 nice=0 active=1/256 refcnt=2
pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x28
pwq 2: cpus=1 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pwq 0: cpus=0 node=0 flags=0x0 nice=0 active=1/256 refcnt=2
pending: wg_packet_encrypt_worker
pool 0: cpus=0 node=0 flags=0x0 nice=0 hung=170s workers=7 idle: 5841 6581 171 6176 6169 9
pool 2: cpus=1 node=0 flags=0x0 nice=0 hung=169s workers=7 idle: 8523 27 5958 23 968 5752
pool 4: cpus=0-1 flags=0x4 nice=0 hung=169s workers=13 idle: 1091 1073 1101
pool 5: cpus=0-1 node=0 flags=0x4 nice=-20 hung=5s workers=5 idle: 52 5085 5924 5923
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 0:
task:kworker/0:0 state:R running task stack:24624 pid:8 ppid:2 flags:0x00004000
Workqueue: events free_obj_work
Call Trace:
context_switch kernel/sched/core.c:5381 [inline]
__schedule+0x1553/0x45a0 kernel/sched/core.c:6700
preempt_schedule_irq+0xbf/0x150 kernel/sched/core.c:7010
irqentry_exit+0x67/0x70 kernel/entry/common.c:438
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:687
RIP: 0010:stack_trace_consume_entry+0x5/0x270 kernel/stacktrace.c:83
Code: 00 00 00 48 3b 4c 24 78 75 09 48 8d 65 f0 5b 41 5e 5d c3 e8 3d 93 12 09 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 00 f3 0f 1e fa 55 <41> 57 41 56 41 55 41 54 53 48 83 ec 18 48 ba 00 00 00 00 00 fc ff
RSP: 0018:ffffc900000d7638 EFLAGS: 00000286
RAX: ffffffff8132a57a RBX: ffffc900000d7700 RCX: ffff88801a26da00
RDX: 0000000000000000 RSI: ffffffff815a3aaa RDI: ffffc900000d7700
RBP: ffffc900000d76d0 R08: ffff88801a26da00 R09: 0000000000000003
R10: 0000000000000004 R11: 0000000000000000 R12: ffffffff815a3aaa
R13: ffffc900000d7a60 R14: ffffffff8175c750 R15: ffffc900000d7648
arch_stack_walk+0x138/0x190 arch/x86/kernel/stacktrace.c:27
stack_trace_save+0xaa/0x100 kernel/stacktrace.c:122
kasan_save_stack mm/kasan/common.c:46 [inline]
kasan_set_track+0x4e/0x70 mm/kasan/common.c:53
kasan_save_free_info+0x2e/0x50 mm/kasan/generic.c:522
____kasan_slab_free+0x126/0x1e0 mm/kasan/common.c:237
kasan_slab_free include/linux/kasan.h:164 [inline]
slab_free_hook mm/slub.c:1811 [inline]
slab_free_freelist_hook+0x130/0x1a0 mm/slub.c:1837
slab_free mm/slub.c:3830 [inline]
kmem_cache_free+0xf8/0x270 mm/slub.c:3852
free_obj_work+0x436/0x5a0 lib/debugobjects.c:332
process_one_work kernel/workqueue.c:2653 [inline]
process_scheduled_works+0xa5d/0x15d0 kernel/workqueue.c:2730
worker_thread+0xa55/0xfc0 kernel/workqueue.c:2811
kthread+0x2fa/0x390 kernel/kthread.c:388
ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
pool 2:
task:kworker/1:4 state:R running task stack:26600 pid:5953 ppid:2 flags:0x00004000
Workqueue: events ovs_dp_masks_rebalance
Call Trace:
context_switch kernel/sched/core.c:5381 [inline]
__schedule+0x1553/0x45a0 kernel/sched/core.c:6700
preempt_schedule_irq+0xbf/0x150 kernel/sched/core.c:7010
irqentry_exit+0x67/0x70 kernel/entry/common.c:438
asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:687
RIP: 0010:queue_delayed_work_on+0x1ab/0x200 kernel/workqueue.c:2011
Code: 4d 85 ed 75 16 e8 55 aa 2f 00 eb 15 e8 4e aa 2f 00 e8 39 da 30 09 4d 85 ed 74 ea e8 3f aa 2f 00 fb 48 c7 44 24 20 0e 36 e0 45 <48> b8 00 00 00 00 00 fc ff df 49 c7 04 06 00 00 00 00 41 c7 44 06
RSP: 0018:ffffc900033b7ae0 EFLAGS: 00000293
RAX: ffffffff81578151 RBX: 0000000000000000 RCX: ffff888026618000
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffffc900033b7bb0 R08: ffffffff911cd597 R09: 1ffffffff2239ab2
R10: dffffc0000000000 R11: fffffbfff2239ab3 R12: ffff888026618001
R13: 0000000000000200 R14: 1ffff92000676f60 R15: 0000000000000000
process_one_work kernel/workqueue.c:2653 [inline]
process_scheduled_works+0xa5d/0x15d0 kernel/workqueue.c:2730
worker_thread+0xa55/0xfc0 kernel/workqueue.c:2811
kthread+0x2fa/0x390 kernel/kthread.c:388
ret_from_fork+0x48/0x80 arch/x86/kernel/process.c:152
ret_from_fork_asm+0x11/0x20 arch/x86/entry/entry_64.S:293
----------------
Code disassembly (best guess):
0: 00 00 add %al,(%rax)
2: 00 48 3b add %cl,0x3b(%rax)
5: 4c 24 78 rex.WR and $0x78,%al
8: 75 09 jne 0x13
a: 48 8d 65 f0 lea -0x10(%rbp),%rsp
e: 5b pop %rbx
f: 41 5e pop %r14
11: 5d pop %rbp
12: c3 ret
13: e8 3d 93 12 09 call 0x9129355
18: 66 2e 0f 1f 84 00 00 cs nopw 0x0(%rax,%rax,1)
1f: 00 00 00
22: 0f 1f 00 nopl (%rax)
25: f3 0f 1e fa endbr64
29: 55 push %rbp
* 2a: 41 57 push %r15 <-- trapping instruction
2c: 41 56 push %r14
2e: 41 55 push %r13
30: 41 54 push %r12
32: 53 push %rbx
33: 48 83 ec 18 sub $0x18,%rsp
37: 48 rex.W
38: ba 00 00 00 00 mov $0x0,%edx
3d: 00 fc add %bh,%ah
3f: ff .byte 0xff