BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 187s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=37 refcnt=38
    in-flight: 798:rht_deferred_worker ,5947:ovs_dp_masks_rebalance
    pending: 4*nsim_dev_hwstats_traffic_work, 3*psi_avgs_work, page_pool_release_retry, psi_avgs_work, vmstat_shepherd, psi_avgs_work, 5*ovs_dp_masks_rebalance, free_obj_work, delayed_vfree_work, nfc_urelease_event_work, ipvlan_process_multicast, 9*rht_deferred_worker, switchdev_deferred_process_work, 5*rht_deferred_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    in-flight: 44:rht_deferred_worker
    pending: psi_avgs_work, nsim_dev_hwstats_traffic_work
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=27 refcnt=28
    pending: 5*defense_work_handler, 2*br_multicast_gc_work, br_fdb_cleanup, 2*br_multicast_gc_work, 2*br_fdb_cleanup, 3*br_multicast_gc_work, 2*br_fdb_cleanup, 5*br_multicast_gc_work, br_fdb_cleanup, 2*br_multicast_gc_work, 2*br_fdb_cleanup
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: br_multicast_gc_work, br_fdb_cleanup, defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: flush_memcg_stats_dwork, toggle_allocation_gate
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 7486:macvlan_process_broadcast
    pending: macvlan_process_broadcast
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=5 refcnt=6
    in-flight: 68:nsim_dev_trap_report_work ,7480:nsim_dev_trap_report_work
    pending: 3*nsim_dev_trap_report_work
workqueue events_freezable: flags=0x104
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: update_balloon_stats_func
workqueue events_power_efficient: flags=0x180
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9
    pending: gc_worker, 3*nf_flow_offload_work_gc, hash_mac4_gc, 3*check_lifetime
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5
    in-flight: 5990:neigh_periodic_work
    pending: wg_ratelimiter_gc_entries, 2*neigh_managed_work
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=3 MAYDAY
    pending: kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue mld: flags=0x40108
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: addrconf_verify_work
    inactive: addrconf_verify_work
workqueue krdsd: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4
    in-flight: 8056:rds_connect_worker
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=33
    in-flight: 3408(RESCUER):batadv_dat_purge
    inactive: batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, 5*batadv_mcast_mla_update, batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, batadv_tt_purge, 3*batadv_purge_orig, 4*batadv_tt_purge
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 13:wg_queued_expired_zero_key_material
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 43:wg_packet_handshake_send_worker wg_packet_handshake_send_worker
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 7477:wg_queued_expired_zero_key_material ,6691:wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 5967:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 2993:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 7472:wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 7469:wg_packet_handshake_send_worker
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 1110:wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 36:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 9:wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 12:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 7475:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
workqueue bond1: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_mii_monitor
    inactive: bond_alb_monitor
workqueue bond1: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_alb_monitor
    inactive: bond_mii_monitor
workqueue bond0: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_mii_monitor
    inactive: bond_alb_monitor
workqueue bond1: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_mii_monitor
    inactive: bond_alb_monitor
workqueue bond2: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_alb_monitor
    inactive: bond_mii_monitor
workqueue bond3: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_mii_monitor
    inactive: bond_alb_monitor
workqueue bond5: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_mii_monitor
    inactive: bond_alb_monitor
workqueue bond6: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_alb_monitor
    inactive: bond_mii_monitor
workqueue bond0: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_mii_monitor
    inactive: bond_alb_monitor
workqueue bond2: flags=0x2000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=6 MAYDAY
    pending: bond_alb_monitor
    inactive: bond_mii_monitor
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=188s workers=5 idle: 5908 5907
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=10s workers=7 idle: 5925 5900 5899 14341 14340
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=17 manager: 16261
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 2:
task:kworker/0:2     state:R  running task     stack:23672 pid:798   tgid:798   ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: events rht_deferred_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5260 [inline]
 __schedule+0x14ea/0x5050 kernel/sched/core.c:6867
 preempt_schedule_irq+0x4d/0xa0 kernel/sched/core.c:7194
 irqentry_exit+0x597/0x620 kernel/entry/common.c:216
 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697
RIP: 0010:qlink_to_object mm/kasan/quarantine.c:140 [inline]
RIP: 0010:qlink_free mm/kasan/quarantine.c:145 [inline]
RIP: 0010:qlist_free_all+0x6d/0x100 mm/kasan/quarantine.c:179
Code: 0c 04 49 8b 44 04 08 48 89 c2 48 ff ca a8 01 48 0f 44 d1 0f b6 42 33 c1 e0 18 3d 00 00 00 f5 49 0f 45 d5 4c 8b 62 08 49 8b 2f <49> 63 84 24 d8 00 00 00 49 29 c7 4c 89 e7 4c 89 fe e8 5d e4 ff ff
RSP: 0018:ffffc900030777e8 EFLAGS: 00000246
RAX: 00000000f5000000 RBX: ffffc90003077820 RCX: ffffea00014c3740
RDX: ffffea00014c3740 RSI: 0000000000000000 RDI: 0000000000000000
RBP: ffff8880530dd1e0 R08: 0000000000000000 R09: ffffffff822cd57c
R10: dffffc0000000000 R11: fffffbfff1fde34f R12: ffff88813fe26640
R13: 0000000000000000 R14: 0000000000000000 R15: ffff8880530dd1c0
 kasan_quarantine_reduce+0x148/0x160 mm/kasan/quarantine.c:286
 __kasan_slab_alloc+0x22/0x80 mm/kasan/common.c:350
 kasan_slab_alloc include/linux/kasan.h:253 [inline]
 slab_post_alloc_hook mm/slub.c:4953 [inline]
 slab_alloc_node mm/slub.c:5263 [inline]
 __do_kmalloc_node mm/slub.c:5656 [inline]
 __kvmalloc_node_noprof+0x548/0x8d0 mm/slub.c:7140
 bucket_table_alloc+0x63/0x370 lib/rhashtable.c:186
 rhashtable_rehash_alloc+0x8b/0x120 lib/rhashtable.c:367
 rht_deferred_worker+0x511/0x2180 lib/rhashtable.c:-1
 process_one_work kernel/workqueue.c:3257 [inline]
 process_scheduled_works+0xaec/0x17a0 kernel/workqueue.c:3340
 worker_thread+0xda6/0x1360 kernel/workqueue.c:3421
 kthread+0x726/0x8b0 kernel/kthread.c:463
 ret_from_fork+0x51b/0xa40 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>
pool 2:
task:kworker/0:0     state:R  running task     stack:24280 pid:9     tgid:9     ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: wg-crypt-wg0 wg_packet_encrypt_worker
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5260 [inline]
 __schedule+0x14ea/0x5050 kernel/sched/core.c:6867
 preempt_schedule_common+0x82/0xd0 kernel/sched/core.c:7051
 preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
 __raw_spin_unlock_irq include/linux/spinlock_api_smp.h:160 [inline]
 _raw_spin_unlock_irq+0x45/0x50 kernel/locking/spinlock.c:202
 process_one_work kernel/workqueue.c:3226 [inline]
 process_scheduled_works+0x8f2/0x17a0 kernel/workqueue.c:3340
 worker_thread+0xda6/0x1360 kernel/workqueue.c:3421
 kthread+0x726/0x8b0 kernel/kthread.c:463
 ret_from_fork+0x51b/0xa40 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>
pool 2:
task:kworker/0:6     state:R  running task     stack:24760 pid:5947  tgid:5947  ppid:2      task_flags:0x4208060 flags:0x00080000
Workqueue: events ovs_dp_masks_rebalance
Call Trace:
 <TASK>
 context_switch kernel/sched/core.c:5260 [inline]
 __schedule+0x14ea/0x5050 kernel/sched/core.c:6867
 preempt_schedule_common+0x82/0xd0 kernel/sched/core.c:7051
 preempt_schedule_thunk+0x16/0x30 arch/x86/entry/thunk.S:12
 __raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:152 [inline]
 _raw_spin_unlock_irqrestore+0x74/0x80 kernel/locking/spinlock.c:194
 spin_unlock_irqrestore include/linux/spinlock.h:406 [inline]
 __put_partials+0x10e/0x170 mm/slub.c:3879
 __slab_free+0x294/0x320 mm/slub.c:5956
 qlink_free mm/kasan/quarantine.c:163 [inline]
 qlist_free_all+0x97/0x100 mm/kasan/quarantine.c:179
 kasan_quarantine_reduce+0x148/0x160 mm/kasan/quarantine.c:286
 __kasan_kmalloc+0x22/0xb0 mm/kasan/common.c:406
 kasan_kmalloc include/linux/kasan.h:263 [inline]
 __do_kmalloc_node mm/slub.c:5657 [inline]
 __kmalloc_noprof+0x40c/0x7e0 mm/slub.c:5669
 kmalloc_noprof include/linux/slab.h:961 [inline]
 kmalloc_array_noprof include/linux/slab.h:1003 [inline]
 ovs_flow_masks_rebalance+0xf4/0xc30 net/openvswitch/flow_table.c:1113
 ovs_dp_masks_rebalance+0x56/0xe0 net/openvswitch/datapath.c:2564
 process_one_work kernel/workqueue.c:3257 [inline]
 process_scheduled_works+0xaec/0x17a0 kernel/workqueue.c:3340
 worker_thread+0xda6/0x1360 kernel/workqueue.c:3421
 kthread+0x726/0x8b0 kernel/kthread.c:463
 ret_from_fork+0x51b/0xa40 arch/x86/kernel/process.c:158
 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246
 </TASK>
----------------
Code disassembly (best guess):
   0:	0c 04                	or     $0x4,%al
   2:	49 8b 44 04 08       	mov    0x8(%r12,%rax,1),%rax
   7:	48 89 c2             	mov    %rax,%rdx
   a:	48 ff ca             	dec    %rdx
   d:	a8 01                	test   $0x1,%al
   f:	48 0f 44 d1          	cmove  %rcx,%rdx
  13:	0f b6 42 33          	movzbl 0x33(%rdx),%eax
  17:	c1 e0 18             	shl    $0x18,%eax
  1a:	3d 00 00 00 f5       	cmp    $0xf5000000,%eax
  1f:	49 0f 45 d5          	cmovne %r13,%rdx
  23:	4c 8b 62 08          	mov    0x8(%rdx),%r12
  27:	49 8b 2f             	mov    (%r15),%rbp
* 2a:	49 63 84 24 d8 00 00 	movslq 0xd8(%r12),%rax <-- trapping instruction
  31:	00
  32:	49 29 c7             	sub    %rax,%r15
  35:	4c 89 e7             	mov    %r12,%rdi
  38:	4c 89 fe             	mov    %r15,%rsi
  3b:	e8 5d e4 ff ff       	call   0xffffe49d