BUG: workqueue lockup - pool cpus=0 node=0 flags=0x0 nice=0 stuck for 42s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=8 refcnt=9
    in-flight: 856:drm_fb_helper_damage_work for 42s
    pending: 3*nsim_dev_hwstats_traffic_work, psi_avgs_work, ovs_dp_masks_rebalance, vmstat_shepherd, drm_fb_helper_damage_work
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=17 refcnt=18
    in-flight: 5001:nsim_fib_event_work for 35s ,2818:nsim_fib_event_work for 2s ,5215:nsim_fib_event_work for 2s ,5003:nsim_fib_event_work for 35s
    pending: netdev_rx_mode_work, 4*nsim_dev_hwstats_traffic_work, delayed_vfree_work, 2*psi_avgs_work, 5*ovs_dp_masks_rebalance
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=10 refcnt=11
    pending: 10*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=8 refcnt=9
    in-flight: 5008:cfg80211_wiphy_work for 25s cfg80211_wiphy_work ,3246:cfg80211_wiphy_work for 43s cfg80211_wiphy_work ,904:cfg80211_wiphy_work for 34s cfg80211_wiphy_work ,5004:cfg80211_wiphy_work for 45s cfg80211_wiphy_work
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=13 refcnt=14
    in-flight: 15:nsim_dev_trap_report_work for 5s ,4834:cfg80211_wiphy_work for 47s cfg80211_wiphy_work ,5370:nsim_dev_trap_report_work for 3s ,5002:cfg80211_wiphy_work for 36s cfg80211_wiphy_work ,1223:nsim_dev_trap_report_work for 12s ,12:nsim_dev_trap_report_work for 0s
    pending: flush_memcg_stats_dwork, toggle_allocation_gate, 3*nsim_dev_trap_report_work
workqueue events_power_efficient: flags=0x182
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=7 refcnt=8
    in-flight: 40:crda_timeout_work for 47s
    pending: neigh_periodic_work, wg_ratelimiter_gc_entries, 2*neigh_managed_work, do_cache_clean, fb_flashcursor
workqueue netns: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4
    in-flight: 39:cleanup_net for 54s
workqueue mm_percpu_wq: flags=0x108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue mld: flags=0x40108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 5009:mld_dad_work for 42s
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=58
    pending: mld_dad_work
    inactive: 52*mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=280 MAYDAY
    in-flight: 2782(RESCUER):addrconf_dad_work for 36s
    inactive: 265*addrconf_dad_work, 5*addrconf_verify_work, 4*addrconf_dad_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=76 MAYDAY
    in-flight: 2817(RESCUER):batadv_tt_purge for 0s
    pending: mayday_cursor_func
    inactive: 3*batadv_mcast_mla_update, 2*batadv_purge_orig, 5*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 4*batadv_iv_send_outstanding_bat_ogm_packet, batadv_mcast_mla_update, batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, batadv_mcast_mla_update, 2*batadv_tt_purge, 3*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_tt_purge, batadv_mcast_mla_update, batadv_iv_send_outstanding_bat_ogm_packet, batadv_tt_purge, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_iv_send_outstanding_bat_ogm_packet, 6*batadv_purge_orig, batadv_tt_purge, 4*batadv_mcast_mla_update
, 4*batadv_iv_send_outstanding_bat_ogm_packet
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 5000:wg_packet_handshake_send_worker for 7s
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 3040:wg_packet_handshake_send_worker for 9s
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    in-flight: 4887:wg_packet_handshake_receive_worker for 42s
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=42s workers=8 idle: 11 9 4344 4716 4780
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=2s workers=8 idle: 24 26 4749 5005
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=15 manager: 5371
Showing backtraces of busy workers in stalled worker pools:
pool 2:
task:kworker/0:7     state:R  running task     stack:0     pid:5009  tgid:5009  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: mld mld_dad_work
Call trace:
 __switch_to+0x2b0/0x6e0 arch/arm64/kernel/process.c:810 (T)
 context_switch kernel/sched/core.c:5388 [inline]
 __schedule+0x1bec/0x2dcc kernel/sched/core.c:7189
 preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7370
 preempt_schedule+0x60/0x78 kernel/sched/core.c:7394
 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:457
 __raw_read_unlock_bh include/linux/rwlock_api_smp.h:283 [inline]
 _raw_read_unlock_bh+0x3c/0x4c kernel/locking/spinlock.c:288
 ipv6_get_lladdr+0x284/0x3c0 net/ipv6/addrconf.c:1942
 mld_newpack+0x36c/0x7b8 net/ipv6/mcast.c:1789
 add_grhead+0x6c/0x25c net/ipv6/mcast.c:1886
 add_grec+0xd08/0x1240 net/ipv6/mcast.c:2025
 mld_send_initial_cr+0x164/0x2f8 net/ipv6/mcast.c:2268
 mld_dad_work+0x40/0x1d8 net/ipv6/mcast.c:2294
 process_one_work kernel/workqueue.c:3314 [inline]
 process_scheduled_works+0x79c/0x1098 kernel/workqueue.c:3397
 worker_thread+0x754/0xba0 kernel/workqueue.c:3478
 kthread+0x2f8/0x3c8 kernel/kthread.c:436
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:842
pool 2:
task:kworker/0:6     state:R  running task     stack:0     pid:4887  tgid:4887  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: wg-kex-wg1 wg_packet_handshake_receive_worker
Call trace:
 __switch_to+0x2b0/0x6e0 arch/arm64/kernel/process.c:810 (T)
 context_switch kernel/sched/core.c:5388 [inline]
 __schedule+0x1bec/0x2dcc kernel/sched/core.c:7189
 preempt_schedule_common+0xd4/0x190 kernel/sched/core.c:7370
 preempt_schedule+0x60/0x78 kernel/sched/core.c:7394
 __local_bh_enable_ip+0x20c/0x35c kernel/softirq.c:457
 local_bh_enable include/linux/bottom_half.h:33 [inline]
 rcu_read_unlock_bh include/linux/rcupdate.h:912 [inline]
 mod_peer_timer+0x21c/0x25c drivers/net/wireguard/timers.c:38
 wg_timers_any_authenticated_packet_traversal+0x98/0xc0 drivers/net/wireguard/timers.c:218
 wg_packet_send_handshake_response+0x120/0x260 drivers/net/wireguard/send.c:99
 wg_receive_handshake_packet drivers/net/wireguard/receive.c:154 [inline]
 wg_packet_handshake_receive_worker+0x550/0xcb8 drivers/net/wireguard/receive.c:213
 process_one_work kernel/workqueue.c:3314 [inline]
 process_scheduled_works+0x79c/0x1098 kernel/workqueue.c:3397
 worker_thread+0x754/0xba0 kernel/workqueue.c:3478
 kthread+0x2f8/0x3c8 kernel/kthread.c:436
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:842
pool 2:
task:kworker/0:2     state:R  running task     stack:0     pid:856   tgid:856   ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events drm_fb_helper_damage_work
Call trace:
 __switch_to+0x2b0/0x6e0 arch/arm64/kernel/process.c:810 (T)
 context_switch kernel/sched/core.c:5388 [inline]
 __schedule+0x1bec/0x2dcc kernel/sched/core.c:7189
 __schedule_loop kernel/sched/core.c:7268 [inline]
 schedule+0x140/0x218 kernel/sched/core.c:7283
 schedule_timeout+0x13c/0x28c kernel/time/sleep_timeout.c:99
 drm_crtc_wait_one_vblank+0x198/0x43c drivers/gpu/drm/drm_vblank.c:1316
 drm_client_modeset_wait_for_vblank+0xc4/0xec drivers/gpu/drm/drm_client_modeset.c:1331
 drm_fb_helper_fb_dirty drivers/gpu/drm/drm_fb_helper.c:237 [inline]
 drm_fb_helper_damage_work+0x100/0x5d0 drivers/gpu/drm/drm_fb_helper.c:275
 process_one_work kernel/workqueue.c:3314 [inline]
 process_scheduled_works+0x79c/0x1098 kernel/workqueue.c:3397
 worker_thread+0x754/0xba0 kernel/workqueue.c:3478
 kthread+0x2f8/0x3c8 kernel/kthread.c:436
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:842