====================================================== WARNING: possible circular locking dependency detected syzkaller #0 Tainted: G L ------------------------------------------------------ kworker/u32:8/1165 is trying to acquire lock: ffffffff8e9771c0 (pcpu_alloc_mutex){+.+.}-{4:4}, at: pcpu_alloc_noprof+0xe87/0x1cc0 mm/percpu.c:1788 but task is already holding lock: ffffffff8e692940 (wq_pool_mutex){+.+.}-{4:4}, at: apply_wqattrs_lock kernel/workqueue.c:5305 [inline] ffffffff8e692940 (wq_pool_mutex){+.+.}-{4:4}, at: __alloc_workqueue+0x858/0x19f0 kernel/workqueue.c:5866 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #9 (wq_pool_mutex){+.+.}-{4:4}: __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1b10 kernel/locking/mutex.c:820 apply_wqattrs_lock kernel/workqueue.c:5305 [inline] __alloc_workqueue+0x858/0x19f0 kernel/workqueue.c:5866 alloc_workqueue_noprof+0xd2/0x200 kernel/workqueue.c:5918 padata_alloc+0xc7/0x400 kernel/padata.c:964 pcrypt_init_padata+0x19/0x100 crypto/pcrypt.c:331 pcrypt_init+0x72/0xf0 crypto/pcrypt.c:356 do_one_initcall+0x121/0x750 init/main.c:1392 do_initcall_level init/main.c:1454 [inline] do_initcalls init/main.c:1470 [inline] do_basic_setup init/main.c:1490 [inline] kernel_init_freeable+0x6ea/0x7b0 init/main.c:1703 kernel_init+0x1f/0x1e0 init/main.c:1593 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #8 (cpu_hotplug_lock){++++}-{0:0}: percpu_down_read_internal include/linux/percpu-rwsem.h:53 [inline] percpu_down_read include/linux/percpu-rwsem.h:77 [inline] cpus_read_lock+0x42/0x170 kernel/cpu.c:490 static_key_slow_inc+0x12/0x30 kernel/jump_label.c:190 fl_create+0x7fb/0xd10 net/ipv6/ip6_flowlabel.c:452 ipv6_flowlabel_get net/ipv6/ip6_flowlabel.c:632 [inline] ipv6_flowlabel_opt+0x519/0x2d40 net/ipv6/ip6_flowlabel.c:735 do_ipv6_setsockopt+0x1926/0x43b0 net/ipv6/ipv6_sockglue.c:928 ipv6_setsockopt+0xcb/0x170 net/ipv6/ipv6_sockglue.c:965 tcp_setsockopt+0xa7/0x100 net/ipv4/tcp.c:4185 do_sock_setsockopt+0xf3/0x1d0 net/socket.c:2381 __sys_setsockopt+0x119/0x190 net/socket.c:2406 __do_sys_setsockopt net/socket.c:2412 [inline] __se_sys_setsockopt net/socket.c:2409 [inline] __ia32_sys_setsockopt+0xbc/0x160 net/socket.c:2409 do_syscall_32_irqs_on arch/x86/entry/syscall_32.c:83 [inline] __do_fast_syscall_32+0xe7/0x950 arch/x86/entry/syscall_32.c:307 do_fast_syscall_32+0x32/0x70 arch/x86/entry/syscall_32.c:332 entry_SYSENTER_compat_after_hwframe+0x84/0x8e -> #7 (sk_lock-AF_INET6){+.+.}-{0:0}: lock_sock_nested+0x41/0xf0 net/core/sock.c:3783 lock_sock include/net/sock.h:1713 [inline] inet_shutdown+0x67/0x410 net/ipv4/af_inet.c:915 nbd_mark_nsock_dead+0xae/0x5c0 drivers/block/nbd.c:318 recv_work+0x5fb/0x8c0 drivers/block/nbd.c:1021 process_one_work+0xa0e/0x1980 kernel/workqueue.c:3302 process_scheduled_works kernel/workqueue.c:3385 [inline] worker_thread+0x5ef/0xe50 kernel/workqueue.c:3466 kthread+0x370/0x450 kernel/kthread.c:436 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #6 (&nsock->tx_lock){+.+.}-{4:4}: __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1b10 kernel/locking/mutex.c:820 nbd_handle_cmd drivers/block/nbd.c:1143 [inline] nbd_queue_rq+0x428/0x1080 drivers/block/nbd.c:1207 blk_mq_dispatch_rq_list+0x422/0x1e70 block/blk-mq.c:2148 __blk_mq_do_dispatch_sched block/blk-mq-sched.c:168 [inline] blk_mq_do_dispatch_sched block/blk-mq-sched.c:182 [inline] __blk_mq_sched_dispatch_requests+0xcea/0x1620 block/blk-mq-sched.c:307 blk_mq_sched_dispatch_requests+0xd7/0x1c0 block/blk-mq-sched.c:329 blk_mq_run_hw_queue+0x23c/0x670 block/blk-mq.c:2386 blk_mq_dispatch_list+0x51d/0x1360 block/blk-mq.c:2949 blk_mq_flush_plug_list block/blk-mq.c:2997 [inline] blk_mq_flush_plug_list+0x130/0x600 block/blk-mq.c:2969 __blk_flush_plug+0x2c4/0x4b0 block/blk-core.c:1230 blk_finish_plug block/blk-core.c:1257 [inline] __submit_bio+0x584/0x6c0 block/blk-core.c:649 __submit_bio_noacct_mq block/blk-core.c:722 [inline] submit_bio_noacct_nocheck+0x543/0xbf0 block/blk-core.c:753 submit_bio_noacct+0xd18/0x2000 block/blk-core.c:884 blk_crypto_submit_bio include/linux/blk-crypto.h:203 [inline] submit_bh_wbc+0x681/0x890 fs/buffer.c:2737 submit_bh fs/buffer.c:2742 [inline] block_read_full_folio+0x4c8/0x8e0 fs/buffer.c:2358 filemap_read_folio+0xfc/0x3b0 mm/filemap.c:2502 do_read_cache_folio+0x2d7/0x6b0 mm/filemap.c:4107 read_mapping_folio include/linux/pagemap.h:1017 [inline] read_part_sector+0xd1/0x370 block/partitions/core.c:724 adfspart_check_ICS+0x91/0x7d0 block/partitions/acorn.c:356 check_partition block/partitions/core.c:143 [inline] blk_add_partitions block/partitions/core.c:591 [inline] bdev_disk_changed+0x7a3/0x1250 block/partitions/core.c:695 blkdev_get_whole+0x187/0x290 block/bdev.c:756 bdev_open+0x2c7/0xe40 block/bdev.c:965 blkdev_open+0x34e/0x4f0 block/fops.c:697 do_dentry_open+0x6d8/0x1660 fs/open.c:947 vfs_open+0x82/0x3f0 fs/open.c:1079 do_open fs/namei.c:4699 [inline] path_openat+0x208c/0x31a0 fs/namei.c:4858 do_file_open+0x20e/0x430 fs/namei.c:4887 do_sys_openat2+0x10d/0x1e0 fs/open.c:1364 do_sys_open fs/open.c:1370 [inline] __do_sys_openat fs/open.c:1386 [inline] __se_sys_openat fs/open.c:1381 [inline] __x64_sys_openat+0x12d/0x210 fs/open.c:1381 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x10b/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x77/0x7f -> #5 (&cmd->lock){+.+.}-{4:4}: __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1b10 kernel/locking/mutex.c:820 nbd_queue_rq+0xba/0x1080 drivers/block/nbd.c:1199 blk_mq_dispatch_rq_list+0x422/0x1e70 block/blk-mq.c:2148 __blk_mq_do_dispatch_sched block/blk-mq-sched.c:168 [inline] blk_mq_do_dispatch_sched block/blk-mq-sched.c:182 [inline] __blk_mq_sched_dispatch_requests+0xcea/0x1620 block/blk-mq-sched.c:307 blk_mq_sched_dispatch_requests+0xd7/0x1c0 block/blk-mq-sched.c:329 blk_mq_run_hw_queue+0x23c/0x670 block/blk-mq.c:2386 blk_mq_dispatch_list+0x51d/0x1360 block/blk-mq.c:2949 blk_mq_flush_plug_list block/blk-mq.c:2997 [inline] blk_mq_flush_plug_list+0x130/0x600 block/blk-mq.c:2969 __blk_flush_plug+0x2c4/0x4b0 block/blk-core.c:1230 blk_finish_plug block/blk-core.c:1257 [inline] __submit_bio+0x584/0x6c0 block/blk-core.c:649 __submit_bio_noacct_mq block/blk-core.c:722 [inline] submit_bio_noacct_nocheck+0x543/0xbf0 block/blk-core.c:753 submit_bio_noacct+0xd18/0x2000 block/blk-core.c:884 blk_crypto_submit_bio include/linux/blk-crypto.h:203 [inline] submit_bh_wbc+0x681/0x890 fs/buffer.c:2737 submit_bh fs/buffer.c:2742 [inline] block_read_full_folio+0x4c8/0x8e0 fs/buffer.c:2358 filemap_read_folio+0xfc/0x3b0 mm/filemap.c:2502 do_read_cache_folio+0x2d7/0x6b0 mm/filemap.c:4107 read_mapping_folio include/linux/pagemap.h:1017 [inline] read_part_sector+0xd1/0x370 block/partitions/core.c:724 adfspart_check_ICS+0x91/0x7d0 block/partitions/acorn.c:356 check_partition block/partitions/core.c:143 [inline] blk_add_partitions block/partitions/core.c:591 [inline] bdev_disk_changed+0x7a3/0x1250 block/partitions/core.c:695 blkdev_get_whole+0x187/0x290 block/bdev.c:756 bdev_open+0x2c7/0xe40 block/bdev.c:965 blkdev_open+0x34e/0x4f0 block/fops.c:697 do_dentry_open+0x6d8/0x1660 fs/open.c:947 vfs_open+0x82/0x3f0 fs/open.c:1079 do_open fs/namei.c:4699 [inline] path_openat+0x208c/0x31a0 fs/namei.c:4858 do_file_open+0x20e/0x430 fs/namei.c:4887 do_sys_openat2+0x10d/0x1e0 fs/open.c:1364 do_sys_open fs/open.c:1370 [inline] __do_sys_openat fs/open.c:1386 [inline] __se_sys_openat fs/open.c:1381 [inline] __x64_sys_openat+0x12d/0x210 fs/open.c:1381 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x10b/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x77/0x7f -> #4 (set->srcu){.+.+}-{0:0}: srcu_lock_sync include/linux/srcu.h:199 [inline] __synchronize_srcu+0xa2/0x300 kernel/rcu/srcutree.c:1481 blk_mq_wait_quiesce_done block/blk-mq.c:284 [inline] blk_mq_wait_quiesce_done block/blk-mq.c:281 [inline] blk_mq_quiesce_queue block/blk-mq.c:304 [inline] blk_mq_quiesce_queue+0x149/0x1c0 block/blk-mq.c:299 elevator_switch+0x17b/0x7e0 block/elevator.c:576 elevator_change+0x352/0x530 block/elevator.c:681 elevator_set_default+0x29e/0x360 block/elevator.c:754 blk_register_queue+0x48e/0x630 block/blk-sysfs.c:987 __add_disk+0x73f/0xe40 block/genhd.c:528 add_disk_fwnode+0x118/0x5c0 block/genhd.c:597 add_disk include/linux/blkdev.h:794 [inline] nbd_dev_add+0x77a/0xb10 drivers/block/nbd.c:1984 nbd_init+0x291/0x2b0 drivers/block/nbd.c:2692 do_one_initcall+0x121/0x750 init/main.c:1392 do_initcall_level init/main.c:1454 [inline] do_initcalls init/main.c:1470 [inline] do_basic_setup init/main.c:1490 [inline] kernel_init_freeable+0x6ea/0x7b0 init/main.c:1703 kernel_init+0x1f/0x1e0 init/main.c:1593 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #3 (&q->elevator_lock){+.+.}-{4:4}: __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1b10 kernel/locking/mutex.c:820 elevator_change+0x1bc/0x530 block/elevator.c:679 elevator_set_none+0x92/0xf0 block/elevator.c:769 blk_mq_elv_switch_none block/blk-mq.c:5131 [inline] __blk_mq_update_nr_hw_queues block/blk-mq.c:5176 [inline] blk_mq_update_nr_hw_queues+0x4c1/0x15f0 block/blk-mq.c:5241 nbd_start_device+0x1a6/0xbd0 drivers/block/nbd.c:1489 nbd_genl_connect+0xff2/0x1a40 drivers/block/nbd.c:2239 genl_family_rcv_msg_doit+0x214/0x300 net/netlink/genetlink.c:1114 genl_family_rcv_msg net/netlink/genetlink.c:1194 [inline] genl_rcv_msg+0x560/0x800 net/netlink/genetlink.c:1209 netlink_rcv_skb+0x159/0x420 net/netlink/af_netlink.c:2550 genl_rcv+0x28/0x40 net/netlink/genetlink.c:1218 netlink_unicast_kernel net/netlink/af_netlink.c:1318 [inline] netlink_unicast+0x585/0x850 net/netlink/af_netlink.c:1344 netlink_sendmsg+0x8b0/0xda0 net/netlink/af_netlink.c:1894 sock_sendmsg_nosec net/socket.c:787 [inline] __sock_sendmsg net/socket.c:802 [inline] ____sys_sendmsg+0x9e1/0xb70 net/socket.c:2698 ___sys_sendmsg+0x190/0x1e0 net/socket.c:2752 __sys_sendmsg+0x170/0x220 net/socket.c:2784 do_syscall_32_irqs_on arch/x86/entry/syscall_32.c:83 [inline] __do_fast_syscall_32+0xe7/0x950 arch/x86/entry/syscall_32.c:307 do_fast_syscall_32+0x32/0x70 arch/x86/entry/syscall_32.c:332 entry_SYSENTER_compat_after_hwframe+0x84/0x8e -> #2 (&q->q_usage_counter(io)#50){++++}-{0:0}: blk_alloc_queue+0x610/0x790 block/blk-core.c:461 blk_mq_alloc_queue+0x174/0x290 block/blk-mq.c:4450 __blk_mq_alloc_disk+0x29/0x120 block/blk-mq.c:4497 nbd_dev_add+0x492/0xb10 drivers/block/nbd.c:1954 nbd_init+0x291/0x2b0 drivers/block/nbd.c:2692 do_one_initcall+0x121/0x750 init/main.c:1392 do_initcall_level init/main.c:1454 [inline] do_initcalls init/main.c:1470 [inline] do_basic_setup init/main.c:1490 [inline] kernel_init_freeable+0x6ea/0x7b0 init/main.c:1703 kernel_init+0x1f/0x1e0 init/main.c:1593 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #1 (fs_reclaim){+.+.}-{0:0}: __fs_reclaim_acquire mm/page_alloc.c:4325 [inline] fs_reclaim_acquire+0xc4/0x100 mm/page_alloc.c:4339 might_alloc include/linux/sched/mm.h:317 [inline] prepare_alloc_pages+0x166/0x5f0 mm/page_alloc.c:4995 __alloc_frozen_pages_noprof+0x19a/0x2bc0 mm/page_alloc.c:5215 __alloc_pages_noprof+0xb/0x110 mm/page_alloc.c:5260 __alloc_pages_node_noprof include/linux/gfp.h:289 [inline] alloc_pages_node_noprof include/linux/gfp.h:316 [inline] pcpu_alloc_pages mm/percpu-vm.c:95 [inline] pcpu_populate_chunk+0x118/0x990 mm/percpu-vm.c:285 pcpu_alloc_noprof+0x881/0x1cc0 mm/percpu.c:1876 iommu_dma_init_fq_percpu drivers/iommu/dma-iommu.c:321 [inline] iommu_dma_init_fq+0x1da/0x7e0 drivers/iommu/dma-iommu.c:347 iommu_dma_init_domain drivers/iommu/dma-iommu.c:720 [inline] iommu_setup_dma_ops+0x11fa/0x1650 drivers/iommu/dma-iommu.c:2149 bus_iommu_probe+0x264/0x550 drivers/iommu/iommu.c:1963 iommu_device_register+0x1af/0x280 drivers/iommu/iommu.c:286 intel_iommu_init+0x2020/0x2970 drivers/iommu/intel/iommu.c:2659 pci_iommu_init+0x3c/0x80 arch/x86/kernel/pci-dma.c:174 do_one_initcall+0x121/0x750 init/main.c:1392 do_initcall_level init/main.c:1454 [inline] do_initcalls init/main.c:1470 [inline] do_basic_setup init/main.c:1490 [inline] kernel_init_freeable+0x6ea/0x7b0 init/main.c:1703 kernel_init+0x1f/0x1e0 init/main.c:1593 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #0 (pcpu_alloc_mutex){+.+.}-{4:4}: check_prev_add kernel/locking/lockdep.c:3165 [inline] check_prevs_add kernel/locking/lockdep.c:3284 [inline] validate_chain kernel/locking/lockdep.c:3908 [inline] __lock_acquire+0x14b8/0x2630 kernel/locking/lockdep.c:5237 lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b1/0x370 kernel/locking/lockdep.c:5825 __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1b10 kernel/locking/mutex.c:820 pcpu_alloc_noprof+0xe87/0x1cc0 mm/percpu.c:1788 alloc_and_link_pwqs kernel/workqueue.c:5599 [inline] __alloc_workqueue+0x8ff/0x19f0 kernel/workqueue.c:5868 alloc_workqueue_noprof+0xd2/0x200 kernel/workqueue.c:5918 __rds_conn_create+0x1234/0x25a0 net/rds/connection.c:274 rds_conn_create+0x44/0x60 net/rds/connection.c:367 rds_tcp_accept_one+0x2fa/0xe80 net/rds/tcp_listen.c:226 rds_tcp_accept_worker+0x41/0x60 net/rds/tcp.c:530 process_one_work+0xa0e/0x1980 kernel/workqueue.c:3302 process_scheduled_works kernel/workqueue.c:3385 [inline] worker_thread+0x5ef/0xe50 kernel/workqueue.c:3466 kthread+0x370/0x450 kernel/kthread.c:436 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 other info that might help us debug this: Chain exists of: pcpu_alloc_mutex --> cpu_hotplug_lock --> wq_pool_mutex Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(wq_pool_mutex); lock(cpu_hotplug_lock); lock(wq_pool_mutex); lock(pcpu_alloc_mutex); *** DEADLOCK *** 4 locks held by kworker/u32:8/1165: #0: ffff88802599a140 ((wq_completion)krdsd){+.+.}-{0:0}, at: process_one_work+0x12d6/0x1980 kernel/workqueue.c:3277 #1: ffffc90006ddfd08 ((work_completion)(&rtn->rds_tcp_accept_w)){+.+.}-{0:0}, at: process_one_work+0x973/0x1980 kernel/workqueue.c:3278 #2: ffff88802a040660 (&rtn->rds_tcp_accept_lock){+.+.}-{4:4}, at: rds_tcp_accept_one+0xb1/0xe80 net/rds/tcp_listen.c:172 #3: ffffffff8e692940 (wq_pool_mutex){+.+.}-{4:4}, at: apply_wqattrs_lock kernel/workqueue.c:5305 [inline] #3: ffffffff8e692940 (wq_pool_mutex){+.+.}-{4:4}, at: __alloc_workqueue+0x858/0x19f0 kernel/workqueue.c:5866 stack backtrace: CPU: 1 UID: 0 PID: 1165 Comm: kworker/u32:8 Tainted: G L syzkaller #0 PREEMPT(full) Tainted: [L]=SOFTLOCKUP Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.3-debian-1.16.3-2 04/01/2014 Workqueue: krdsd rds_tcp_accept_worker Call Trace: __dump_stack lib/dump_stack.c:94 [inline] dump_stack_lvl+0x100/0x190 lib/dump_stack.c:120 print_circular_bug.cold+0x178/0x1c7 kernel/locking/lockdep.c:2043 check_noncircular+0x146/0x160 kernel/locking/lockdep.c:2175 check_prev_add kernel/locking/lockdep.c:3165 [inline] check_prevs_add kernel/locking/lockdep.c:3284 [inline] validate_chain kernel/locking/lockdep.c:3908 [inline] __lock_acquire+0x14b8/0x2630 kernel/locking/lockdep.c:5237 lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b1/0x370 kernel/locking/lockdep.c:5825 __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1b10 kernel/locking/mutex.c:820 pcpu_alloc_noprof+0xe87/0x1cc0 mm/percpu.c:1788 alloc_and_link_pwqs kernel/workqueue.c:5599 [inline] __alloc_workqueue+0x8ff/0x19f0 kernel/workqueue.c:5868 alloc_workqueue_noprof+0xd2/0x200 kernel/workqueue.c:5918 __rds_conn_create+0x1234/0x25a0 net/rds/connection.c:274 rds_conn_create+0x44/0x60 net/rds/connection.c:367 rds_tcp_accept_one+0x2fa/0xe80 net/rds/tcp_listen.c:226 rds_tcp_accept_worker+0x41/0x60 net/rds/tcp.c:530 process_one_work+0xa0e/0x1980 kernel/workqueue.c:3302 process_scheduled_works kernel/workqueue.c:3385 [inline] worker_thread+0x5ef/0xe50 kernel/workqueue.c:3466 kthread+0x370/0x450 kernel/kthread.c:436 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245