====================================================== WARNING: possible circular locking dependency detected syzkaller #0 Tainted: G L ------------------------------------------------------ kworker/2:1H/1300 is trying to acquire lock: ffff888077c42a68 (&nsock->tx_lock){+.+.}-{4:4}, at: nbd_handle_cmd drivers/block/nbd.c:1143 [inline] ffff888077c42a68 (&nsock->tx_lock){+.+.}-{4:4}, at: nbd_queue_rq+0x428/0x1080 drivers/block/nbd.c:1207 but task is already holding lock: ffff888027367178 (&cmd->lock){+.+.}-{4:4}, at: nbd_queue_rq+0xba/0x1080 drivers/block/nbd.c:1199 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #6 (&cmd->lock){+.+.}-{4:4}: lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b9/0x370 kernel/locking/lockdep.c:5825 __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1bd0 kernel/locking/mutex.c:821 nbd_queue_rq+0xba/0x1080 drivers/block/nbd.c:1199 blk_mq_dispatch_rq_list+0x422/0x1e70 block/blk-mq.c:2117 __blk_mq_do_dispatch_sched block/blk-mq-sched.c:168 [inline] blk_mq_do_dispatch_sched block/blk-mq-sched.c:182 [inline] __blk_mq_sched_dispatch_requests+0xcea/0x1620 block/blk-mq-sched.c:307 blk_mq_sched_dispatch_requests+0xd7/0x1c0 block/blk-mq-sched.c:329 blk_mq_run_work_fn+0x135/0x380 block/blk-mq.c:2531 process_one_work+0xa23/0x1940 kernel/workqueue.c:3322 process_scheduled_works kernel/workqueue.c:3405 [inline] worker_thread+0x5ef/0xe50 kernel/workqueue.c:3486 kthread+0x370/0x450 kernel/kthread.c:436 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #5 (set->srcu){.+.+}-{0:0}: lock_sync kernel/locking/lockdep.c:5916 [inline] lock_sync+0x9d/0x110 kernel/locking/lockdep.c:5904 srcu_lock_sync include/linux/srcu.h:199 [inline] __synchronize_srcu+0xa2/0x300 kernel/rcu/srcutree.c:1481 blk_mq_wait_quiesce_done block/blk-mq.c:284 [inline] blk_mq_wait_quiesce_done block/blk-mq.c:281 [inline] blk_mq_quiesce_queue block/blk-mq.c:304 [inline] blk_mq_quiesce_queue+0x149/0x1c0 block/blk-mq.c:299 elevator_switch+0x17b/0x7e0 block/elevator.c:576 elevator_change+0x352/0x530 block/elevator.c:681 elevator_set_default+0x29e/0x360 block/elevator.c:754 blk_register_queue+0x48e/0x630 block/blk-sysfs.c:992 __add_disk+0x73f/0xe40 block/genhd.c:524 add_disk_fwnode+0x118/0x480 block/genhd.c:593 add_disk include/linux/blkdev.h:800 [inline] nbd_dev_add+0x77a/0xb10 drivers/block/nbd.c:2021 nbd_init+0x291/0x2b0 drivers/block/nbd.c:2729 do_one_initcall+0x11d/0x700 init/main.c:1347 do_initcall_level init/main.c:1409 [inline] do_initcalls init/main.c:1425 [inline] do_basic_setup init/main.c:1445 [inline] kernel_init_freeable+0x6ea/0x7b0 init/main.c:1658 kernel_init+0x1f/0x1e0 init/main.c:1548 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #4 (&q->elevator_lock){+.+.}-{4:4}: lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b9/0x370 kernel/locking/lockdep.c:5825 __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1bd0 kernel/locking/mutex.c:821 elevator_change+0x1bc/0x530 block/elevator.c:679 elv_iosched_store+0x3be/0x460 block/elevator.c:826 queue_attr_store+0x25f/0x2f0 block/blk-sysfs.c:906 sysfs_kf_write+0xf2/0x150 fs/sysfs/file.c:145 kernfs_fop_write_iter+0x3e0/0x5f0 fs/kernfs/file.c:345 new_sync_write fs/read_write.c:595 [inline] vfs_write+0x6ac/0x1050 fs/read_write.c:687 ksys_write+0x12a/0x250 fs/read_write.c:739 do_syscall_32_irqs_on arch/x86/entry/syscall_32.c:83 [inline] __do_fast_syscall_32+0xe7/0x970 arch/x86/entry/syscall_32.c:307 do_fast_syscall_32+0x32/0x70 arch/x86/entry/syscall_32.c:332 entry_SYSENTER_compat_after_hwframe+0x84/0x8e -> #3 (&q->q_usage_counter(io)#66){++++}-{0:0}: lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b9/0x370 kernel/locking/lockdep.c:5825 blk_alloc_queue+0x610/0x790 block/blk-core.c:504 blk_mq_alloc_queue+0x174/0x290 block/blk-mq.c:4420 __blk_mq_alloc_disk+0x29/0x120 block/blk-mq.c:4467 null_add_dev+0xdf8/0x1df0 drivers/block/null_blk/main.c:2008 null_create_dev drivers/block/null_blk/main.c:2106 [inline] null_init+0x3a4/0x480 drivers/block/null_blk/main.c:2178 do_one_initcall+0x11d/0x700 init/main.c:1347 do_initcall_level init/main.c:1409 [inline] do_initcalls init/main.c:1425 [inline] do_basic_setup init/main.c:1445 [inline] kernel_init_freeable+0x6ea/0x7b0 init/main.c:1658 kernel_init+0x1f/0x1e0 init/main.c:1548 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 -> #2 (fs_reclaim){+.+.}-{0:0}: lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b9/0x370 kernel/locking/lockdep.c:5825 __fs_reclaim_acquire mm/page_alloc.c:4329 [inline] fs_reclaim_acquire+0xc4/0x100 mm/page_alloc.c:4343 might_alloc include/linux/sched/mm.h:317 [inline] slab_pre_alloc_hook mm/slub.c:4537 [inline] slab_alloc_node mm/slub.c:4897 [inline] __kmalloc_cache_noprof+0x63/0x6c0 mm/slub.c:5485 _kmalloc_noprof include/linux/slab.h:988 [inline] _kzalloc_noprof include/linux/slab.h:1309 [inline] tcp_sendmsg_fastopen+0x24d/0x750 net/ipv4/tcp.c:1067 tcp_sendmsg_locked+0x276d/0x44d0 net/ipv4/tcp.c:1181 tcp_sendmsg+0x2e/0x50 net/ipv4/tcp.c:1451 inet_sendmsg+0xb9/0x140 net/ipv4/af_inet.c:865 sock_sendmsg_nosec net/socket.c:775 [inline] __sock_sendmsg net/socket.c:790 [inline] ____sys_sendmsg+0x9c9/0xbe0 net/socket.c:2684 ___sys_sendmsg+0x190/0x1e0 net/socket.c:2738 __sys_sendmsg+0x160/0x210 net/socket.c:2770 do_syscall_32_irqs_on arch/x86/entry/syscall_32.c:83 [inline] __do_fast_syscall_32+0xe7/0x970 arch/x86/entry/syscall_32.c:307 do_fast_syscall_32+0x32/0x70 arch/x86/entry/syscall_32.c:332 entry_SYSENTER_compat_after_hwframe+0x84/0x8e -> #1 (sk_lock-AF_INET-NBD){+.+.}-{0:0}: lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b9/0x370 kernel/locking/lockdep.c:5825 lock_sock_nested+0x41/0xf0 net/core/sock.c:3827 lock_sock include/net/sock.h:1713 [inline] inet_shutdown+0x67/0x410 net/ipv4/af_inet.c:913 nbd_mark_nsock_dead+0xae/0x5c0 drivers/block/nbd.c:318 sock_shutdown+0x16b/0x200 drivers/block/nbd.c:411 nbd_clear_sock drivers/block/nbd.c:1464 [inline] nbd_config_put+0x1eb/0x750 drivers/block/nbd.c:1488 nbd_release+0xb7/0x190 drivers/block/nbd.c:1793 blkdev_put_whole+0xb0/0xf0 block/bdev.c:724 bdev_release+0x47f/0x6d0 block/bdev.c:1147 blkdev_release+0x15/0x20 block/fops.c:680 __fput+0x3ff/0xb50 fs/file_table.c:512 task_work_run+0x150/0x240 kernel/task_work.c:233 resume_user_mode_work include/linux/resume_user_mode.h:50 [inline] __exit_to_user_mode_loop kernel/entry/common.c:70 [inline] exit_to_user_mode_loop+0x1d8/0x6f0 kernel/entry/common.c:101 __exit_to_user_mode_prepare include/linux/irq-entry-common.h:207 [inline] syscall_exit_to_user_mode_prepare include/linux/irq-entry-common.h:230 [inline] syscall_exit_to_user_mode include/linux/entry-common.h:318 [inline] __do_fast_syscall_32+0x701/0x970 arch/x86/entry/syscall_32.c:310 do_fast_syscall_32+0x32/0x70 arch/x86/entry/syscall_32.c:332 entry_SYSENTER_compat_after_hwframe+0x84/0x8e -> #0 (&nsock->tx_lock){+.+.}-{4:4}: check_prev_add+0xeb/0xe60 kernel/locking/lockdep.c:3165 check_prevs_add kernel/locking/lockdep.c:3284 [inline] validate_chain kernel/locking/lockdep.c:3908 [inline] __lock_acquire+0x136c/0x1a40 kernel/locking/lockdep.c:5237 lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b9/0x370 kernel/locking/lockdep.c:5825 __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1bd0 kernel/locking/mutex.c:821 nbd_handle_cmd drivers/block/nbd.c:1143 [inline] nbd_queue_rq+0x428/0x1080 drivers/block/nbd.c:1207 blk_mq_dispatch_rq_list+0x422/0x1e70 block/blk-mq.c:2117 __blk_mq_do_dispatch_sched block/blk-mq-sched.c:168 [inline] blk_mq_do_dispatch_sched block/blk-mq-sched.c:182 [inline] __blk_mq_sched_dispatch_requests+0xcea/0x1620 block/blk-mq-sched.c:307 blk_mq_sched_dispatch_requests+0xd7/0x1c0 block/blk-mq-sched.c:329 blk_mq_run_work_fn+0x135/0x380 block/blk-mq.c:2531 process_one_work+0xa23/0x1940 kernel/workqueue.c:3322 process_scheduled_works kernel/workqueue.c:3405 [inline] worker_thread+0x5ef/0xe50 kernel/workqueue.c:3486 kthread+0x370/0x450 kernel/kthread.c:436 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 other info that might help us debug this: Chain exists of: &nsock->tx_lock --> set->srcu --> &cmd->lock Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&cmd->lock); lock(set->srcu); lock(&cmd->lock); lock(&nsock->tx_lock); *** DEADLOCK *** 4 locks held by kworker/2:1H/1300: #0: ffff888020876140 ((wq_completion)kblockd){+.+.}-{0:0}, at: process_one_work+0x12b1/0x1940 kernel/workqueue.c:3297 #1: ffffc90006c2fd08 ((work_completion)(&(&hctx->run_work)->work)){+.+.}-{0:0}, at: process_one_work+0x988/0x1940 kernel/workqueue.c:3298 #2: ffff8880269f8498 (set->srcu){.+.+}-{0:0}, at: srcu_lock_acquire include/linux/srcu.h:187 [inline] #2: ffff8880269f8498 (set->srcu){.+.+}-{0:0}, at: srcu_read_lock include/linux/srcu.h:294 [inline] #2: ffff8880269f8498 (set->srcu){.+.+}-{0:0}, at: blk_mq_run_work_fn+0x127/0x380 block/blk-mq.c:2531 #3: ffff888027367178 (&cmd->lock){+.+.}-{4:4}, at: nbd_queue_rq+0xba/0x1080 drivers/block/nbd.c:1199 stack backtrace: CPU: 2 UID: 0 PID: 1300 Comm: kworker/2:1H Tainted: G L syzkaller #0 PREEMPT(full) Tainted: [L]=SOFTLOCKUP Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.3-debian-1.16.3-2 04/01/2014 Workqueue: kblockd blk_mq_run_work_fn Call Trace: __dump_stack lib/dump_stack.c:94 [inline] dump_stack_lvl+0x100/0x190 lib/dump_stack.c:120 print_circular_bug.cold+0x178/0x1c7 kernel/locking/lockdep.c:2043 check_noncircular+0x146/0x160 kernel/locking/lockdep.c:2175 check_prev_add+0xeb/0xe60 kernel/locking/lockdep.c:3165 check_prevs_add kernel/locking/lockdep.c:3284 [inline] validate_chain kernel/locking/lockdep.c:3908 [inline] __lock_acquire+0x136c/0x1a40 kernel/locking/lockdep.c:5237 lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x1b9/0x370 kernel/locking/lockdep.c:5825 __mutex_lock_common kernel/locking/mutex.c:646 [inline] __mutex_lock+0x1a4/0x1bd0 kernel/locking/mutex.c:821 nbd_handle_cmd drivers/block/nbd.c:1143 [inline] nbd_queue_rq+0x428/0x1080 drivers/block/nbd.c:1207 blk_mq_dispatch_rq_list+0x422/0x1e70 block/blk-mq.c:2117 __blk_mq_do_dispatch_sched block/blk-mq-sched.c:168 [inline] blk_mq_do_dispatch_sched block/blk-mq-sched.c:182 [inline] __blk_mq_sched_dispatch_requests+0xcea/0x1620 block/blk-mq-sched.c:307 blk_mq_sched_dispatch_requests+0xd7/0x1c0 block/blk-mq-sched.c:329 blk_mq_run_work_fn+0x135/0x380 block/blk-mq.c:2531 process_one_work+0xa23/0x1940 kernel/workqueue.c:3322 process_scheduled_works kernel/workqueue.c:3405 [inline] worker_thread+0x5ef/0xe50 kernel/workqueue.c:3486 kthread+0x370/0x450 kernel/kthread.c:436 ret_from_fork+0x72b/0xd50 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:245 blk_print_req_error: 10 callbacks suppressed I/O error, dev nbd1, sector 0 op 0x0:(READ) flags 0x800000 phys_seg 1 prio class 2 buffer_io_error: 10 callbacks suppressed Buffer I/O error on dev nbd1, logical block 0, async page read