BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 51s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=12 refcnt=13
    in-flight: 6632:nsim_fib_event_work nsim_fib_event_work ,10:nsim_fib_event_work nsim_fib_event_work ,2375:nsim_fib_event_work
    pending: 2*nsim_dev_hwstats_traffic_work, psi_avgs_work, vmstat_shepherd, drm_fb_helper_damage_work, nsim_dev_hwstats_traffic_work, rht_deferred_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=13 refcnt=14
    in-flight: 24:nsim_fib_event_work ,26:nsim_fib_event_work
    pending: 2*nsim_dev_hwstats_traffic_work, psi_avgs_work, ovs_dp_masks_rebalance, rht_deferred_worker, psi_avgs_work, 5*ovs_dp_masks_rebalance
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    pending: 2*defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=4 refcnt=5
    pending: 4*defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 1561:toggle_allocation_gate
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: flush_memcg_stats_dwork
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=10 refcnt=11
    in-flight: 41:cfg80211_wiphy_work cfg80211_wiphy_work ,345:nsim_dev_trap_report_work ,242:cfg80211_wiphy_work cfg80211_wiphy_work ,13:linkwatch_event ,5320:nsim_dev_trap_report_work ,3478:cfg80211_wiphy_work cfg80211_wiphy_work
    pending: nsim_dev_trap_report_work
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=4
    in-flight: 731:cfg80211_wiphy_work cfg80211_wiphy_work
    pending: nsim_dev_trap_report_work
workqueue events_power_efficient: flags=0x182
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=12 refcnt=13
    in-flight: 5812:reg_check_chans_work ,5342:crda_timeout_work ,2011:check_lifetime ,4562:check_lifetime ,3285:neigh_periodic_work ,5167:check_lifetime ,6839:check_lifetime
    pending: fb_flashcursor, 2*neigh_managed_work, neigh_periodic_work, gc_worker
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=4
    in-flight: 12:wg_ratelimiter_gc_entries
    pending: check_lifetime, do_cache_clean
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=3 MAYDAY
    pending: kfree_rcu_monitor
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=4 refcnt=6
    in-flight: 6(RESCUER):kfree_rcu_work kfree_rcu_work kfree_rcu_work kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue writeback: flags=0x4a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=4 MAYDAY
    in-flight: 34(RESCUER):wb_workfn
workqueue mld: flags=0x40108
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=104
    pending: mld_ifc_work
    inactive: 6*mld_ifc_work, 2*mld_dad_work, 12*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 12*mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, 4*mld_ifc_work, 2*mld_dad_work, 7*mld_ifc_work, 2*mld_dad_work, 3*mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, mld_ifc_work, mld_dad_work, 9*mld_ifc_work, 2*mld_dad_work, mld_ifc_work, mld_dad_work, 18*mld_ifc_work, mld_dad_work, 2*mld_ifc_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=383
    in-flight: 2023:addrconf_dad_work
    inactive: 373*addrconf_dad_work, 6*addrconf_verify_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=42
    in-flight: 4322(RESCUER):batadv_tt_purge
    inactive: batadv_bla_periodic_work, batadv_dat_purge, batadv_mcast_mla_update, batadv_tt_purge, 6*batadv_iv_send_outstanding_bat_ogm_packet, batadv_bla_periodic_work, batadv_dat_purge, 2*batadv_purge_orig, 2*batadv_tt_purge, 3*batadv_iv_send_outstanding_bat_ogm_packet, batadv_tt_purge, batadv_mcast_mla_update, batadv_iv_send_outstanding_bat_ogm_packet, batadv_bla_periodic_work, 2*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_mcast_mla_update, 2*batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_purge_orig, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge
workqueue hci1: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    pending: hci_cmd_timeout
workqueue hci1: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    in-flight: 53:hci_cmd_sync_work
workqueue hci2: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    pending: hci_cmd_work
workqueue hci2: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    in-flight: 6160:hci_cmd_sync_work
workqueue hci3: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    pending: hci_cmd_work
workqueue hci3: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    in-flight: 6693:hci_cmd_sync_work
workqueue hci4: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    pending: hci_cmd_timeout
workqueue hci4: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    in-flight: 6702:hci_cmd_sync_work
workqueue hci5: flags=0x20012
  pwq 9: cpus=0-1 node=0 flags=0x6 nice=-20 active=1 refcnt=4
    pending: hci_cmd_sync_work
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 6825:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 2248:wg_packet_handshake_send_worker wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 6781:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_tx_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 6682:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 6833:wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 9:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=2 refcnt=3
    in-flight: 6680:wg_packet_handshake_receive_worker
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 6836:wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg2: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 6840:wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x2 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker, wg_packet_encrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    in-flight: 42:wg_packet_handshake_send_worker
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
pool 2: cpus=0 node=0 flags=0x2 nice=0 hung=15s workers=9 manager: 6837
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=51s workers=8 idle: 6826 6681 6683 1540 6649 6226
pool 8: cpus=0-1 flags=0x6 nice=0 hung=0s workers=23 manager: 6834
pool 9: cpus=0-1 node=0 flags=0x6 nice=-20 hung=5s workers=5 manager: 6646
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 6:
task:kworker/1:0     state:R  running task     stack:0     pid:24    tgid:24    ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:742 (T)
 context_switch kernel/sched/core.c:5256 [inline]
 __schedule+0x1250/0x2a7c kernel/sched/core.c:6863
 preempt_schedule_irq+0x80/0x188 kernel/sched/core.c:7190
 raw_irqentry_exit_cond_resched+0x30/0x44 kernel/entry/common.c:173
 irqentry_exit+0x1b0/0x308 kernel/entry/common.c:216
 exit_to_kernel_mode+0x10/0x1c arch/arm64/kernel/entry-common.c:58
 __el1_irq arch/arm64/kernel/entry-common.c:500 [inline]
 el1_interrupt+0x4c/0x60 arch/arm64/kernel/entry-common.c:510
 el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:515
 el1h_64_irq+0x6c/0x70 arch/arm64/kernel/entry.S:592
 __debug_check_no_obj_freed lib/debugobjects.c:1120 [inline] (P)
 debug_check_no_obj_freed+0x70/0x470 lib/debugobjects.c:1129 (P)
 free_pages_prepare mm/page_alloc.c:1413 [inline]
 __free_frozen_pages+0x4b8/0xcb0 mm/page_alloc.c:2943
 free_frozen_pages+0x14/0x20 mm/page_alloc.c:2981
 __free_slab+0x100/0x1a4 mm/slub.c:3317
 free_slab+0x28/0x160 mm/slub.c:3340
 discard_slab+0x48/0x98 mm/slub.c:3346
 __slab_free+0x208/0x23c mm/slub.c:6004
 do_slab_free mm/slub.c:6618 [inline]
 ___cache_free+0x150/0x170 mm/slub.c:6737
 qlink_free+0x48/0x94 mm/kasan/quarantine.c:163
 qlist_free_all+0x44/0xb8 mm/kasan/quarantine.c:179
 kasan_quarantine_reduce+0x118/0x124 mm/kasan/quarantine.c:286
 __kasan_slab_alloc+0x2c/0x88 mm/kasan/common.c:350
 kasan_slab_alloc include/linux/kasan.h:253 [inline]
 slab_post_alloc_hook mm/slub.c:4953 [inline]
 slab_alloc_node mm/slub.c:5263 [inline]
 __kmalloc_cache_noprof+0x34c/0x698 mm/slub.c:5771
 kmalloc_noprof include/linux/slab.h:957 [inline]
 kzalloc_noprof include/linux/slab.h:1094 [inline]
 nsim_fib4_rt_create drivers/net/netdevsim/fib.c:280 [inline]
 nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:426 [inline]
 nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline]
 nsim_fib_event+0x3c4/0x6fc0 drivers/net/netdevsim/fib.c:884
 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493
 process_one_work+0x7c0/0x1558 kernel/workqueue.c:3257
 process_scheduled_works kernel/workqueue.c:3340 [inline]
 worker_thread+0x990/0xed8 kernel/workqueue.c:3421
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844
pool 6:
task:kworker/1:1     state:R  running task     stack:0     pid:26    tgid:26    ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:742 (T)
 context_switch kernel/sched/core.c:5256 [inline]
 __schedule+0x1250/0x2a7c kernel/sched/core.c:6863
 preempt_schedule_irq+0x80/0x188 kernel/sched/core.c:7190
 raw_irqentry_exit_cond_resched+0x30/0x44 kernel/entry/common.c:173
 irqentry_exit+0x1b0/0x308 kernel/entry/common.c:216
 exit_to_kernel_mode+0x10/0x1c arch/arm64/kernel/entry-common.c:58
 __el1_irq arch/arm64/kernel/entry-common.c:500 [inline]
 el1_interrupt+0x4c/0x60 arch/arm64/kernel/entry-common.c:510
 el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:515
 el1h_64_irq+0x6c/0x70 arch/arm64/kernel/entry.S:592
 __sanitizer_cov_trace_pc+0x8/0x84 kernel/kcov.c:210 (P)
 page_ext_iter_begin include/linux/page_ext.h:132 [inline]
 __page_table_check_zero+0x15c/0x3d4 mm/page_table_check.c:139
 page_table_check_free include/linux/page_table_check.h:43 [inline]
 free_pages_prepare mm/page_alloc.c:1407 [inline]
 __free_frozen_pages+0xb9c/0xcb0 mm/page_alloc.c:2943
 free_frozen_pages+0x14/0x20 mm/page_alloc.c:2981
 __free_slab+0x100/0x1a4 mm/slub.c:3317
 free_slab+0x28/0x160 mm/slub.c:3340
 discard_slab+0x48/0x98 mm/slub.c:3346
 __slab_free+0x208/0x23c mm/slub.c:6004
 do_slab_free mm/slub.c:6618 [inline]
 ___cache_free+0x150/0x170 mm/slub.c:6737
 qlink_free+0x48/0x94 mm/kasan/quarantine.c:163
 qlist_free_all+0x44/0xb8 mm/kasan/quarantine.c:179
 kasan_quarantine_reduce+0x118/0x124 mm/kasan/quarantine.c:286
 __kasan_slab_alloc+0x2c/0x88 mm/kasan/common.c:350
 kasan_slab_alloc include/linux/kasan.h:253 [inline]
 slab_post_alloc_hook mm/slub.c:4953 [inline]
 slab_alloc_node mm/slub.c:5263 [inline]
 __kmalloc_cache_noprof+0x34c/0x698 mm/slub.c:5771
 kmalloc_noprof include/linux/slab.h:957 [inline]
 kzalloc_noprof include/linux/slab.h:1094 [inline]
 nsim_fib4_rt_create drivers/net/netdevsim/fib.c:280 [inline]
 nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:426 [inline]
 nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline]
 nsim_fib_event+0x3c4/0x6fc0 drivers/net/netdevsim/fib.c:884
 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493
 process_one_work+0x7c0/0x1558 kernel/workqueue.c:3257
 process_scheduled_works kernel/workqueue.c:3340 [inline]
 worker_thread+0x990/0xed8 kernel/workqueue.c:3421
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844