BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 56s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=21 refcnt=22
    in-flight: 10:nsim_fib_event_work for 56s ,4832:nsim_fib_event_work for 58s
    pending: 4*nsim_dev_hwstats_traffic_work, psi_avgs_work, rht_deferred_worker, psi_avgs_work, vmstat_shepherd, 6*ovs_dp_masks_rebalance, nsim_fib_event_work, drm_fb_helper_damage_work, reg_todo, rht_deferred_worker, delayed_vfree_work
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=9 refcnt=10
    in-flight: 867:nsim_fib_event_work for 59s nsim_fib_event_work
    pending: nsim_fib_event_work, ovs_dp_masks_rebalance, 2*psi_avgs_work, 3*ovs_dp_masks_rebalance
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9
    pending: 8*defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: 2*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=20 refcnt=21
    in-flight: 262:cfg80211_wiphy_work for 57s cfg80211_wiphy_work ,15:cfg80211_wiphy_work for 46s ,1188:cfg80211_wiphy_work for 57s cfg80211_wiphy_work ,39:cfg80211_wiphy_work for 57s cfg80211_wiphy_work ,12:cfg80211_wiphy_work for 57s cfg80211_wiphy_work
    pending: 2*nsim_dev_trap_report_work, cfg80211_wiphy_work, toggle_allocation_gate, nsim_dev_trap_report_work, macvlan_process_broadcast, flush_memcg_stats_dwork, 2*macvlan_process_broadcast, nsim_dev_trap_report_work, macvlan_process_broadcast
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=6 refcnt=7
    in-flight: 3341:cfg80211_wiphy_work for 47s
    pending: macvlan_process_broadcast, crng_reseed, macvlan_process_broadcast, cfg80211_wiphy_work, macvlan_process_broadcast
workqueue events_power_efficient: flags=0x182
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=8 refcnt=9
    in-flight: 4869:neigh_periodic_work for 34s
    pending: neigh_periodic_work, fb_flashcursor, wg_ratelimiter_gc_entries, neigh_managed_work, do_cache_clean, neigh_managed_work, check_lifetime
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: gc_worker
workqueue netns: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4
    in-flight: 40:cleanup_net for 60s
workqueue mm_percpu_wq: flags=0x108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue writeback: flags=0x4a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4 MAYDAY
    in-flight: 33(RESCUER):wb_workfn for 44s
    pending: mayday_cursor_func
workqueue mld: flags=0x40108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=76
    in-flight: 4924:mld_dad_work for 56s
    inactive: 2*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, 2*mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, 3*mld_dad_work, mld_ifc_work, 2*mld_dad_work, 4*mld_ifc_work, 2*mld_dad_work, 2*mld_ifc_work, 47*mld_dad_work
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=152
    in-flight: 4945:mld_ifc_work for 13s
    inactive: 6*mld_ifc_work, 2*mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, mld_dad_work, mld_ifc_work, 2*mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 5*mld_ifc_work, mld_dad_work, 9*mld_ifc_work, 2*mld_dad_work, 5*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, 5*mld_dad_work, mld_ifc_work, mld_dad_work, 7*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 6*mld_ifc_work, mld_dad_work, 3*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, 4*mld_dad_work, 6*mld_ifc_work, 50*mld_dad_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=139 MAYDAY
    in-flight: 2817(RESCUER):addrconf_dad_work for 0s
    pending: mayday_cursor_func
    inactive: 132*addrconf_dad_work, addrconf_verify_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=51 MAYDAY
    in-flight: 2853(RESCUER):batadv_tt_purge for 13s
    pending: mayday_cursor_func
    inactive: 3*batadv_tt_purge, batadv_mcast_mla_update, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_purge_orig, 7*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 7*batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 5*batadv_mcast_mla_update, 2*batadv_tt_purge
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    in-flight: 4981:wg_packet_encrypt_worker for 56s
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    pending: 2*wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=56s workers=7 idle: 4528 9 2840
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=13s workers=6 idle: 4947 24 26 4764
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=9 manager: 4990
Showing backtraces of busy workers in stalled worker pools:
pool 2:
task:kworker/0:6     state:R  running task     stack:0     pid:4981  tgid:4981  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: wg-crypt-wg2 wg_packet_encrypt_worker
Call trace:
 __switch_to+0x2b0/0x6e0 arch/arm64/kernel/process.c:810 (T)
 context_switch kernel/sched/core.c:5388 [inline]
 __schedule+0x1bec/0x2dcc kernel/sched/core.c:7189
 preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7370
 preempt_schedule+0x60/0x78 kernel/sched/core.c:7394
 __kunmap_atomic include/linux/highmem-internal.h:247 [inline]
 sg_miter_stop+0x220/0x2c4 lib/scatterlist.c:941
 chacha20poly1305_crypt_sg_inplace+0x758/0xc04 lib/crypto/chacha20poly1305.c:319
 chacha20poly1305_encrypt_sg_inplace+0x54/0x70 lib/crypto/chacha20poly1305.c:340
 encrypt_packet drivers/net/wireguard/send.c:216 [inline]
 wg_packet_encrypt_worker+0x71c/0x1228 drivers/net/wireguard/send.c:297
 process_one_work kernel/workqueue.c:3314 [inline]
 process_scheduled_works+0x79c/0x1098 kernel/workqueue.c:3397
 worker_thread+0x754/0xba0 kernel/workqueue.c:3478
 kthread+0x2f8/0x3c8 kernel/kthread.c:436
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:842
pool 2:
task:kworker/0:5     state:R  running task     stack:0     pid:4924  tgid:4924  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: mld mld_dad_work
Call trace:
 __switch_to+0x2b0/0x6e0 arch/arm64/kernel/process.c:810 (T)
 context_switch kernel/sched/core.c:5388 [inline]
 __schedule+0x1bec/0x2dcc kernel/sched/core.c:7189
 preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7370
 preempt_schedule+0x60/0x78 kernel/sched/core.c:7394
 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:457
 local_bh_enable include/linux/bottom_half.h:33 [inline]
 rcu_read_unlock_bh include/linux/rcupdate.h:912 [inline]
 __dev_queue_xmit+0x11dc/0x2c18 net/core/dev.c:4905
 dev_queue_xmit include/linux/netdevice.h:3418 [inline]
 neigh_hh_output include/net/neighbour.h:544 [inline]
 neigh_output include/net/neighbour.h:558 [inline]
 ip6_finish_output2+0xcb0/0x1a7c net/ipv6/ip6_output.c:136
 __ip6_finish_output net/ipv6/ip6_output.c:-1 [inline]
 ip6_finish_output+0x230/0x618 net/ipv6/ip6_output.c:219
 NF_HOOK_COND include/linux/netfilter.h:307 [inline]
 ip6_output+0x2d0/0x644 net/ipv6/ip6_output.c:246
 dst_output include/net/dst.h:470 [inline]
 NF_HOOK+0x14c/0x45c include/linux/netfilter.h:318
 mld_sendpack+0x7fc/0xff0 net/ipv6/mcast.c:1855
 mld_send_initial_cr+0x234/0x2f8 net/ipv6/mcast.c:2271
 mld_dad_work+0x40/0x1d8 net/ipv6/mcast.c:2294
 process_one_work kernel/workqueue.c:3314 [inline]
 process_scheduled_works+0x79c/0x1098 kernel/workqueue.c:3397
 worker_thread+0x754/0xba0 kernel/workqueue.c:3478
 kthread+0x2f8/0x3c8 kernel/kthread.c:436
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:842
pool 2:
task:kworker/0:1     state:R  running task     stack:0     pid:10    tgid:10    ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x2b0/0x6e0 arch/arm64/kernel/process.c:810 (T)
 context_switch kernel/sched/core.c:5388 [inline]
 __schedule+0x1bec/0x2dcc kernel/sched/core.c:7189
 __schedule_loop kernel/sched/core.c:7268 [inline]
 schedule+0x140/0x218 kernel/sched/core.c:7283
 schedule_timeout+0x13c/0x28c kernel/time/sleep_timeout.c:99
 schedule_timeout_uninterruptible+0x78/0xbc kernel/time/sleep_timeout.c:158
 msleep+0x3c/0x68 kernel/time/sleep_timeout.c:318
 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline]
 nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline]
 nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline]
 nsim_fib_event+0x53c4/0x5f48 drivers/net/netdevsim/fib.c:889
 nsim_fib_event_work+0x1cc/0x30c drivers/net/netdevsim/fib.c:1493
 process_one_work kernel/workqueue.c:3314 [inline]
 process_scheduled_works+0x79c/0x1098 kernel/workqueue.c:3397
 worker_thread+0x754/0xba0 kernel/workqueue.c:3478
 kthread+0x2f8/0x3c8 kernel/kthread.c:436
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:842
pool 2:
task:kworker/0:4     state:R  running task     stack:0     pid:4832  tgid:4832  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x2b0/0x6e0 arch/arm64/kernel/process.c:810 (T)
 context_switch kernel/sched/core.c:5388 [inline]
 __schedule+0x1bec/0x2dcc kernel/sched/core.c:7189
 __schedule_loop kernel/sched/core.c:7268 [inline]
 schedule+0x140/0x218 kernel/sched/core.c:7283
 schedule_timeout+0x13c/0x28c kernel/time/sleep_timeout.c:99
 schedule_timeout_uninterruptible+0x78/0xbc kernel/time/sleep_timeout.c:158
 msleep+0x3c/0x68 kernel/time/sleep_timeout.c:318
 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline]
 nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline]
 nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline]
 nsim_fib_event+0x53c4/0x5f48 drivers/net/netdevsim/fib.c:889
 nsim_fib_event_work+0x1cc/0x30c drivers/net/netdevsim/fib.c:1493
 process_one_work kernel/workqueue.c:3314 [inline]
 process_scheduled_works+0x79c/0x1098 kernel/workqueue.c:3397
 worker_thread+0x754/0xba0 kernel/workqueue.c:3478
 kthread+0x2f8/0x3c8 kernel/kthread.c:436
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:842