buffer.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/fs/buffer.c
 *
 *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
 */

/*
 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
 *
 * Removed a lot of unnecessary code and simplified things now that
 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
 *
 * Speed up hash, lru, and free list operations.  Use gfp() for allocating
 * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
 *
 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
 *
 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
 */

#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/syscalls.h>
#include <linux/fs.h>
#include <linux/iomap.h>
#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/capability.h>
#include <linux/blkdev.h>
#include <linux/blk-crypto.h>
#include <linux/file.h>
#include <linux/quotaops.h>
#include <linux/highmem.h>
#include <linux/export.h>
#include <linux/backing-dev.h>
#include <linux/writeback.h>
#include <linux/hash.h>
#include <linux/suspend.h>
#include <linux/buffer_head.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/bio.h>
#include <linux/cpu.h>
#include <linux/bitops.h>
#include <linux/mpage.h>
#include <linux/bit_spinlock.h>
#include <linux/folio_batch.h>
#include <linux/sched/mm.h>
#include <trace/events/block.h>
#include <linux/fscrypt.h>
#include <linux/fsverity.h>
#include <linux/sched/isolation.h>

#include "internal.h"

#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)

inline void touch_buffer(struct buffer_head *bh)
{
	trace_block_touch_buffer(bh);
	folio_mark_accessed(bh->b_folio);
}
EXPORT_SYMBOL(touch_buffer);

void __lock_buffer(struct buffer_head *bh)
{
	wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__lock_buffer);

void unlock_buffer(struct buffer_head *bh)
{
	clear_and_wake_up_bit(BH_Lock, &bh->b_state);
}
EXPORT_SYMBOL(unlock_buffer);

/*
 * Returns if the folio has dirty or writeback buffers. If all the buffers
 * are unlocked and clean then the folio_test_dirty information is stale. If
 * any of the buffers are locked, it is assumed they are locked for IO.
 */
void buffer_check_dirty_writeback(struct folio *folio,
				     bool *dirty, bool *writeback)
{
	struct buffer_head *head, *bh;
	*dirty = false;
	*writeback = false;

	BUG_ON(!folio_test_locked(folio));

	head = folio_buffers(folio);
	if (!head)
		return;

	if (folio_test_writeback(folio))
		*writeback = true;

	bh = head;
	do {
		if (buffer_locked(bh))
			*writeback = true;

		if (buffer_dirty(bh))
			*dirty = true;

		bh = bh->b_this_page;
	} while (bh != head);
}

/*
 * Block until a buffer comes unlocked.  This doesn't stop it
 * from becoming locked again - you have to lock it yourself
 * if you want to preserve its state.
 */
void __wait_on_buffer(struct buffer_head * bh)
{
	wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__wait_on_buffer);

static void buffer_io_error(struct buffer_head *bh, char *msg)
{
	if (!test_bit(BH_Quiet, &bh->b_state))
		printk_ratelimited(KERN_ERR
			"Buffer I/O error on dev %pg, logical block %llu%s\n",
			bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
}

/**
 * bio_endio_bh - Discard the bio used to submit a buffer.
 * @bio: The bio.
 * @bhp: Where to return the buffer_head.
 *
 * Call this in your bio_end_io handler to retrieve the buffer_head
 * submitted in bh_submit().  If you did not call bh_submit(), do not
 * call this function; it will return garbage.
 *
 * This function consumes the bio refcount which will probably free the
 * bio.
 *
 * Return: True if the I/O succeeded.
 */
bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp)
{
	bool success = bio->bi_status == BLK_STS_OK;
	struct buffer_head *bh = bio->bi_private;

	if (unlikely(bio_flagged(bio, BIO_QUIET)))
		set_bit(BH_Quiet, &bh->b_state);
	bio_put(bio);

	*bhp = bh;
	return success;
}
EXPORT_SYMBOL(bio_endio_bh);

/**
 * end_buffer_read_sync - Handle buffer reads finishing
 * @bh: The buffer.
 * @uptodate: True if the read was successful.
 *
 * If a buffer is read through a mechanism that isn't bh_submit(), you
 * can call this function to finish the read.
 */
void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
{
	if (uptodate) {
		set_buffer_uptodate(bh);
	} else {
		/* This happens, due to failed read-ahead attempts. */
		clear_buffer_uptodate(bh);
	}
	unlock_buffer(bh);
}
EXPORT_SYMBOL(end_buffer_read_sync);

/**
 * bh_end_read - I/O end handler for reads
 * @bio: The bio being completed.
 *
 * Pass this function to bh_submit() if you're reading into the buffer,
 * unless you need your own special I/O end handler.
 */
void bh_end_read(struct bio *bio)
{
	struct buffer_head *bh;
	bool uptodate = bio_endio_bh(bio, &bh);
	end_buffer_read_sync(bh, uptodate);
}
EXPORT_SYMBOL(bh_end_read);

/**
 * bh_end_write - I/O end handler for writes
 * @bio: The bio being completed.
 *
 * Pass this function to bh_submit() if you're writing from the buffer,
 * unless you need your own special I/O end handler.
 */
void bh_end_write(struct bio *bio)
{
	struct buffer_head *bh;
	bool success = bio_endio_bh(bio, &bh);

	if (success) {
		set_buffer_uptodate(bh);
	} else {
		buffer_io_error(bh, ", lost sync page write");
		mark_buffer_write_io_error(bh);
		clear_buffer_uptodate(bh);
	}
	unlock_buffer(bh);
}
EXPORT_SYMBOL(bh_end_write);

static struct buffer_head *
__find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic)
{
	struct address_space *bd_mapping = bdev->bd_mapping;
	const int blkbits = bd_mapping->host->i_blkbits;
	struct buffer_head *ret = NULL;
	pgoff_t index;
	struct buffer_head *bh;
	struct buffer_head *head;
	struct folio *folio;
	int all_mapped = 1;
	static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);

	index = ((loff_t)block << blkbits) / PAGE_SIZE;
	folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
	if (IS_ERR(folio))
		goto out;

	/*
	 * Folio lock protects the buffers. Callers that cannot block
	 * will fallback to serializing vs try_to_free_buffers() via
	 * the i_private_lock.
	 */
	if (atomic)
		spin_lock(&bd_mapping->i_private_lock);
	else
		folio_lock(folio);

	head = folio_buffers(folio);
	if (!head)
		goto out_unlock;
	/*
	 * Upon a noref migration, the folio lock serializes here;
	 * otherwise bail.
	 */
	if (test_bit_acquire(BH_Migrate, &head->b_state)) {
		WARN_ON(!atomic);
		goto out_unlock;
	}

	bh = head;
	do {
		if (!buffer_mapped(bh))
			all_mapped = 0;
		else if (bh->b_blocknr == block) {
			ret = bh;
			get_bh(bh);
			goto out_unlock;
		}
		bh = bh->b_this_page;
	} while (bh != head);

	/* we might be here because some of the buffers on this page are
	 * not mapped.  This is due to various races between
	 * file io on the block device and getblk.  It gets dealt with
	 * elsewhere, don't buffer_error if we had some unmapped buffers
	 */
	ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
	if (all_mapped && __ratelimit(&last_warned)) {
		printk("__find_get_block_slow() failed. block=%llu, "
		       "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
		       "device %pg blocksize: %d\n",
		       (unsigned long long)block,
		       (unsigned long long)bh->b_blocknr,
		       bh->b_state, bh->b_size, bdev,
		       1 << blkbits);
	}
out_unlock:
	if (atomic)
		spin_unlock(&bd_mapping->i_private_lock);
	else
		folio_unlock(folio);
	folio_put(folio);
out:
	return ret;
}

static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
{
	unsigned long flags;
	struct buffer_head *first;
	struct buffer_head *tmp;
	struct folio *folio;
	int folio_uptodate = 1;

	BUG_ON(!buffer_async_read(bh));

	folio = bh->b_folio;
	if (uptodate) {
		set_buffer_uptodate(bh);
	} else {
		clear_buffer_uptodate(bh);
		buffer_io_error(bh, ", async page read");
	}

	/*
	 * Be _very_ careful from here on. Bad things can happen if
	 * two buffer heads end IO at almost the same time and both
	 * decide that the page is now completely done.
	 */
	first = folio_buffers(folio);
	spin_lock_irqsave(&first->b_uptodate_lock, flags);
	clear_buffer_async_read(bh);
	unlock_buffer(bh);
	tmp = bh;
	do {
		if (!buffer_uptodate(tmp))
			folio_uptodate = 0;
		if (buffer_async_read(tmp)) {
			BUG_ON(!buffer_locked(tmp));
			goto still_busy;
		}
		tmp = tmp->b_this_page;
	} while (tmp != bh);
	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);

	folio_end_read(folio, folio_uptodate);
	return;

still_busy:
	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
}

struct verify_bh_ctx {
	struct work_struct work;
	struct buffer_head *bh;
	struct fsverity_info *vi;
};

static void verify_bh(struct work_struct *work)
{
	struct verify_bh_ctx *ctx =
		container_of(work, struct verify_bh_ctx, work);
	struct buffer_head *bh = ctx->bh;
	bool valid;

	valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size,
				       bh_offset(bh));
	end_buffer_async_read(bh, valid);
	kfree(ctx);
}

/*
 * I/O completion handler for block_read_full_folio() - folios
 * which come unlocked at the end of I/O.
 */
static void bh_end_async_read(struct bio *bio)
{
	struct buffer_head *bh;
	bool uptodate = bio_endio_bh(bio, &bh);
	struct inode *inode = bh->b_folio->mapping->host;
	struct fsverity_info *vi = NULL;

	/* needed by ext4 */
	if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE))
		vi = fsverity_get_info(inode);

	/* Verify (with fsverity) if needed. */
	if (vi && uptodate) {
		struct verify_bh_ctx *ctx = kmalloc_obj(*ctx, GFP_ATOMIC);

		if (ctx) {
			ctx->bh = bh;
			ctx->vi = vi;
			INIT_WORK(&ctx->work, verify_bh);
			fsverity_enqueue_verify_work(&ctx->work);
			return;
		}
		uptodate = false;
	}
	end_buffer_async_read(bh, uptodate);
}

/**
 * bh_end_async_write - I/O end handler for async folio writes
 * @bio: The bio being completed.
 *
 * Pass this function to bh_submit() if you're doing the equivalent of
 * block_write_full_folio().  That is, the folio is unlocked, and will
 * have its writeback flag cleared once all async write buffers have
 * completed.
 */
void bh_end_async_write(struct bio *bio)
{
	struct buffer_head *bh;
	bool success = bio_endio_bh(bio, &bh);
	unsigned long flags;
	struct buffer_head *first;
	struct buffer_head *tmp;
	struct folio *folio;

	BUG_ON(!buffer_async_write(bh));

	folio = bh->b_folio;
	if (success) {
		set_buffer_uptodate(bh);
	} else {
		buffer_io_error(bh, ", lost async page write");
		mark_buffer_write_io_error(bh);
		clear_buffer_uptodate(bh);
	}

	first = folio_buffers(folio);
	spin_lock_irqsave(&first->b_uptodate_lock, flags);

	clear_buffer_async_write(bh);
	unlock_buffer(bh);
	tmp = bh->b_this_page;
	while (tmp != bh) {
		if (buffer_async_write(tmp)) {
			BUG_ON(!buffer_locked(tmp));
			goto still_busy;
		}
		tmp = tmp->b_this_page;
	}
	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
	folio_end_writeback(folio);
	return;

still_busy:
	spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
}
EXPORT_SYMBOL(bh_end_async_write);


/*
 * fs/buffer.c contains helper functions for buffer-backed address space's
 * fsync functions.  A common requirement for buffer-based filesystems is
 * that certain data from the backing blockdev needs to be written out for
 * a successful fsync().  For example, ext2 indirect blocks need to be
 * written back and waited upon before fsync() returns.
 *
 * The functions mmb_mark_buffer_dirty(), mmb_sync(), mmb_has_buffers()
 * and mmb_invalidate() are provided for the management of a list of dependent
 * buffers in mapping_metadata_bhs struct.
 *
 * The locking is a little subtle: The list of buffer heads is protected by
 * the lock in mapping_metadata_bhs so functions coming from bdev mapping
 * (such as try_to_free_buffers()) need to safely get to mapping_metadata_bhs
 * using RCU, grab the lock, verify we didn't race with somebody detaching the
 * bh / moving it to different inode and only then proceeding.
 */

void mmb_init(struct mapping_metadata_bhs *mmb, struct address_space *mapping)
{
	spin_lock_init(&mmb->lock);
	INIT_LIST_HEAD(&mmb->list);
	mmb->mapping = mapping;
}
EXPORT_SYMBOL(mmb_init);

static void __remove_assoc_queue(struct mapping_metadata_bhs *mmb,
			         struct buffer_head *bh)
{
	lockdep_assert_held(&mmb->lock);
	list_del_init(&bh->b_assoc_buffers);
	WARN_ON(!bh->b_mmb);
	bh->b_mmb = NULL;
}

static void remove_assoc_queue(struct buffer_head *bh)
{
	struct mapping_metadata_bhs *mmb;

	/*
	 * The locking dance is ugly here. We need to acquire the lock
	 * protecting the metadata bh list while possibly racing with bh
	 * being removed from the list or moved to a different one.  We
	 * use RCU to pin mapping_metadata_bhs in memory to
	 * opportunistically acquire the lock and then recheck the bh
	 * didn't move under us.
	 */
	while (bh->b_mmb) {
		rcu_read_lock();
		mmb = READ_ONCE(bh->b_mmb);
		if (mmb) {
			spin_lock(&mmb->lock);
			if (bh->b_mmb == mmb)
				__remove_assoc_queue(mmb, bh);
			spin_unlock(&mmb->lock);
		}
		rcu_read_unlock();
	}
}

bool mmb_has_buffers(struct mapping_metadata_bhs *mmb)
{
	return !list_empty(&mmb->list);
}
EXPORT_SYMBOL_GPL(mmb_has_buffers);

/**
 * mmb_sync - write out & wait upon all buffers in a list
 * @mmb: the list of buffers to write
 *
 * Starts I/O against the buffers in the given list and waits upon
 * that I/O. Basically, this is a convenience function for fsync().  @mmb is
 * for a file or directory which needs those buffers to be written for a
 * successful fsync().
 *
 * We have conflicting pressures: we want to make sure that all
 * initially dirty buffers get waited on, but that any subsequently
 * dirtied buffers don't.  After all, we don't want fsync to last
 * forever if somebody is actively writing to the file.
 *
 * Do this in two main stages: first we copy dirty buffers to a
 * temporary inode list, queueing the writes as we go. Then we clean
 * up, waiting for those writes to complete. mark_buffer_dirty_inode()
 * doesn't touch b_assoc_buffers list if b_mmb is not NULL so we are sure the
 * buffer stays on our list until IO completes (at which point it can be
 * reaped).
 */
int mmb_sync(struct mapping_metadata_bhs *mmb)
{
	struct buffer_head *bh;
	int err = 0;
	struct blk_plug plug;
	LIST_HEAD(tmp);

	if (!mmb_has_buffers(mmb))
		return 0;

	blk_start_plug(&plug);

	spin_lock(&mmb->lock);
	while (!list_empty(&mmb->list)) {
		bh = BH_ENTRY(mmb->list.next);
		WARN_ON_ONCE(bh->b_mmb != mmb);
		__remove_assoc_queue(mmb, bh);
		/* Avoid race with mark_buffer_dirty_inode() which does
		 * a lockless check and we rely on seeing the dirty bit */
		smp_mb();
		if (buffer_dirty(bh) || buffer_locked(bh)) {
			list_add(&bh->b_assoc_buffers, &tmp);
			bh->b_mmb = mmb;
			if (buffer_dirty(bh)) {
				get_bh(bh);
				spin_unlock(&mmb->lock);
				/*
				 * Ensure any pending I/O completes so that
				 * write_dirty_buffer() actually writes the
				 * current contents - it is a noop if I/O is
				 * still in flight on potentially older
				 * contents.
				 */
				write_dirty_buffer(bh, REQ_SYNC);

				/*
				 * Kick off IO for the previous mapping. Note
				 * that we will not run the very last mapping,
				 * wait_on_buffer() will do that for us
				 * through sync_buffer().
				 */
				brelse(bh);
				spin_lock(&mmb->lock);
			}
		}
	}

	spin_unlock(&mmb->lock);
	blk_finish_plug(&plug);
	spin_lock(&mmb->lock);

	while (!list_empty(&tmp)) {
		bh = BH_ENTRY(tmp.prev);
		get_bh(bh);
		__remove_assoc_queue(mmb, bh);
		/* Avoid race with mark_buffer_dirty_inode() which does
		 * a lockless check and we rely on seeing the dirty bit */
		smp_mb();
		if (buffer_dirty(bh)) {
			list_add(&bh->b_assoc_buffers, &mmb->list);
			bh->b_mmb = mmb;
		}
		spin_unlock(&mmb->lock);
		wait_on_buffer(bh);
		if (!buffer_uptodate(bh))
			err = -EIO;
		brelse(bh);
		spin_lock(&mmb->lock);
	}
	spin_unlock(&mmb->lock);
	return err;
}
EXPORT_SYMBOL(mmb_sync);

/*
 * Called when we've recently written block `bblock', and it is known that
 * `bblock' was for a buffer_boundary() buffer.  This means that the block at
 * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
 * dirty, schedule it for IO.  So that indirects merge nicely with their data.
 */
void write_boundary_block(struct block_device *bdev,
			sector_t bblock, unsigned blocksize)
{
	struct buffer_head *bh;

	bh = __find_get_block_nonatomic(bdev, bblock + 1, blocksize);
	if (bh) {
		if (buffer_dirty(bh))
			write_dirty_buffer(bh, 0);
		put_bh(bh);
	}
}

void mmb_mark_buffer_dirty(struct buffer_head *bh,
			   struct mapping_metadata_bhs *mmb)
{
	mark_buffer_dirty(bh);
	if (!bh->b_mmb) {
		spin_lock(&mmb->lock);
		/*
		 * For a corrupted filesystem with multiply claimed blocks this
		 * can fail. Avoid corrupting the linked list in that case.
		 */
		if (cmpxchg(&bh->b_mmb, NULL, mmb) != NULL) {
			spin_unlock(&mmb->lock);
			return;
		}
		list_move_tail(&bh->b_assoc_buffers, &mmb->list);
		spin_unlock(&mmb->lock);
	}
}
EXPORT_SYMBOL(mmb_mark_buffer_dirty);

/**
 * block_dirty_folio - Mark a folio as dirty.
 * @mapping: The address space containing this folio.
 * @folio: The folio to mark dirty.
 *
 * Filesystems which use buffer_heads can use this function as their
 * ->dirty_folio implementation.  Some filesystems need to do a little
 * work before calling this function.  Filesystems which do not use
 * buffer_heads should call filemap_dirty_folio() instead.
 *
 * If the folio has buffers, the uptodate buffers are set dirty, to
 * preserve dirty-state coherency between the folio and the buffers.
 * Buffers added to a dirty folio are created dirty.
 *
 * The buffers are dirtied before the folio is dirtied.  There's a small
 * race window in which writeback may see the folio cleanness but not the
 * buffer dirtiness.  That's fine.  If this code were to set the folio
 * dirty before the buffers, writeback could clear the folio dirty flag,
 * see a bunch of clean buffers and we'd end up with dirty buffers/clean
 * folio on the dirty folio list.
 *
 * We use i_private_lock to lock against try_to_free_buffers() while
 * using the folio's buffer list.  This also prevents clean buffers
 * being added to the folio after it was set dirty.
 *
 * Context: May only be called from process context.  Does not sleep.
 * Caller must ensure that @folio cannot be truncated during this call,
 * typically by holding the folio lock or having a page in the folio
 * mapped and holding the page table lock.
 *
 * Return: True if the folio was dirtied; false if it was already dirtied.
 */
bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
{
	struct buffer_head *head;
	bool newly_dirty;

	spin_lock(&mapping->i_private_lock);
	head = folio_buffers(folio);
	if (head) {
		struct buffer_head *bh = head;

		do {
			set_buffer_dirty(bh);
			bh = bh->b_this_page;
		} while (bh != head);
	}
	/*
	 * Lock out page's memcg migration to keep PageDirty
	 * synchronized with per-memcg dirty page counters.
	 */
	newly_dirty = !folio_test_set_dirty(folio);
	spin_unlock(&mapping->i_private_lock);

	if (newly_dirty)
		__folio_mark_dirty(folio, mapping, 1);

	if (newly_dirty)
		__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);

	return newly_dirty;
}
EXPORT_SYMBOL(block_dirty_folio);

/*
 * Invalidate any and all dirty buffers on a given buffers list.  We are
 * probably unmounting the fs, but that doesn't mean we have already
 * done a sync().  Just drop the buffers from the inode list.
 */
void mmb_invalidate(struct mapping_metadata_bhs *mmb)
{
	if (mmb_has_buffers(mmb)) {
		spin_lock(&mmb->lock);
		while (!list_empty(&mmb->list))
			__remove_assoc_queue(mmb, BH_ENTRY(mmb->list.next));
		spin_unlock(&mmb->lock);
	}
}
EXPORT_SYMBOL(mmb_invalidate);

/*
 * Create the appropriate buffers when given a folio for data area and
 * the size of each buffer.. Use the bh->b_this_page linked list to
 * follow the buffers created.  Return NULL if unable to create more
 * buffers.
 *
 * The retry flag is used to differentiate async IO (paging, swapping)
 * which may not fail from ordinary buffer allocations.
 */
struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
					gfp_t gfp)
{
	struct buffer_head *bh, *head;
	long offset;
	struct mem_cgroup *memcg, *old_memcg;

	memcg = get_mem_cgroup_from_folio(folio);
	old_memcg = set_active_memcg(memcg);

	head = NULL;
	offset = folio_size(folio);
	while ((offset -= size) >= 0) {
		bh = alloc_buffer_head(gfp);
		if (!bh)
			goto no_grow;

		bh->b_this_page = head;
		bh->b_blocknr = -1;
		head = bh;

		bh->b_size = size;

		/* Link the buffer to its folio */
		folio_set_bh(bh, folio, offset);
	}
out:
	set_active_memcg(old_memcg);
	mem_cgroup_put(memcg);
	return head;
/*
 * In case anything failed, we just free everything we got.
 */
no_grow:
	if (head) {
		do {
			bh = head;
			head = head->b_this_page;
			free_buffer_head(bh);
		} while (head);
	}

	goto out;
}
EXPORT_SYMBOL_GPL(folio_alloc_buffers);

struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size)
{
	gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;

	return folio_alloc_buffers(page_folio(page), size, gfp);
}
EXPORT_SYMBOL_GPL(alloc_page_buffers);

static inline void link_dev_buffers(struct folio *folio,
		struct buffer_head *head)
{
	struct buffer_head *bh, *tail;

	bh = head;
	do {
		tail = bh;
		bh = bh->b_this_page;
	} while (bh);
	tail->b_this_page = head;
	folio_attach_private(folio, head);
}

static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
{
	sector_t retval = ~((sector_t)0);
	loff_t sz = bdev_nr_bytes(bdev);

	if (sz) {
		unsigned int sizebits = blksize_bits(size);
		retval = (sz >> sizebits);
	}
	return retval;
}

/*
 * Initialise the state of a blockdev folio's buffers.
 */ 
static sector_t folio_init_buffers(struct folio *folio,
		struct block_device *bdev, unsigned size)
{
	struct buffer_head *head = folio_buffers(folio);
	struct buffer_head *bh = head;
	bool uptodate = folio_test_uptodate(folio);
	sector_t block = div_u64(folio_pos(folio), size);
	sector_t end_block = blkdev_max_block(bdev, size);

	do {
		if (!buffer_mapped(bh)) {
			bh->b_private = NULL;
			bh->b_bdev = bdev;
			bh->b_blocknr = block;
			if (uptodate)
				set_buffer_uptodate(bh);
			if (block < end_block)
				set_buffer_mapped(bh);
		}
		block++;
		bh = bh->b_this_page;
	} while (bh != head);

	/*
	 * Caller needs to validate requested block against end of device.
	 */
	return end_block;
}

/*
 * Create the page-cache folio that contains the requested block.
 *
 * This is used purely for blockdev mappings.
 *
 * Returns false if we have a failure which cannot be cured by retrying
 * without sleeping.  Returns true if we succeeded, or the caller should retry.
 */
static bool grow_dev_folio(struct block_device *bdev, sector_t block,
		pgoff_t index, unsigned size, gfp_t gfp)
{
	struct address_space *mapping = bdev->bd_mapping;
	struct folio *folio;
	struct buffer_head *bh;
	sector_t end_block = 0;

	folio = __filemap_get_folio(mapping, index,
			FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
	if (IS_ERR(folio))
		return false;

	bh = folio_buffers(folio);
	if (bh) {
		if (bh->b_size == size) {
			end_block = folio_init_buffers(folio, bdev, size);
			goto unlock;
		}

		/*
		 * Retrying may succeed; for example the folio may finish
		 * writeback, or buffers may be cleaned.  This should not
		 * happen very often; maybe we have old buffers attached to
		 * this blockdev's page cache and we're trying to change
		 * the block size?
		 */
		if (!try_to_free_buffers(folio)) {
			end_block = ~0ULL;
			goto unlock;
		}
	}

	bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT);
	if (!bh)
		goto unlock;

	/*
	 * Link the folio to the buffers and initialise them.  Take the
	 * lock to be atomic wrt __find_get_block(), which does not
	 * run under the folio lock.
	 */
	spin_lock(&mapping->i_private_lock);
	link_dev_buffers(folio, bh);
	end_block = folio_init_buffers(folio, bdev, size);
	spin_unlock(&mapping->i_private_lock);
unlock:
	folio_unlock(folio);
	folio_put(folio);
	return block < end_block;
}

/*
 * Create buffers for the specified block device block's folio.  If
 * that folio was dirty, the buffers are set dirty also.  Returns false
 * if we've hit a permanent error.
 */
static bool grow_buffers(struct block_device *bdev, sector_t block,
		unsigned size, gfp_t gfp)
{
	loff_t pos;

	/*
	 * Check for a block which lies outside our maximum possible
	 * pagecache index.
	 */
	if (check_mul_overflow(block, (sector_t)size, &pos) || pos > MAX_LFS_FILESIZE) {
		printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n",
			__func__, (unsigned long long)block,
			bdev);
		return false;
	}

	/* Create a folio with the proper size buffers */
	return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp);
}

static struct buffer_head *
__getblk_slow(struct block_device *bdev, sector_t block,
	     unsigned size, gfp_t gfp)
{
	bool blocking = gfpflags_allow_blocking(gfp);

	if (WARN_ON_ONCE(!IS_ALIGNED(size, bdev_logical_block_size(bdev)))) {
		printk(KERN_ERR "getblk(): block size %d not aligned to logical block size %d\n",
		       size, bdev_logical_block_size(bdev));
		return NULL;
	}

	for (;;) {
		struct buffer_head *bh;

		if (!grow_buffers(bdev, block, size, gfp))
			return NULL;

		if (blocking)
			bh = __find_get_block_nonatomic(bdev, block, size);
		else
			bh = __find_get_block(bdev, block, size);
		if (bh)
			return bh;
	}
}

/*
 * The relationship between dirty buffers and dirty pages:
 *
 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
 * the page is tagged dirty in the page cache.
 *
 * At all times, the dirtiness of the buffers represents the dirtiness of
 * subsections of the page.  If the page has buffers, the page dirty bit is
 * merely a hint about the true dirty state.
 *
 * When a page is set dirty in its entirety, all its buffers are marked dirty
 * (if the page has buffers).
 *
 * When a buffer is marked dirty, its page is dirtied, but the page's other
 * buffers are not.
 *
 * Also.  When blockdev buffers are explicitly read with bread(), they
 * individually become uptodate.  But their backing page remains not
 * uptodate - even if all of its buffers are uptodate.  A subsequent
 * block_read_full_folio() against that folio will discover all the uptodate
 * buffers, will set the folio uptodate and will perform no I/O.
 */

/**
 * mark_buffer_dirty - mark a buffer_head as needing writeout
 * @bh: the buffer_head to mark dirty
 *
 * mark_buffer_dirty() will set the dirty bit against the buffer, then set
 * its backing page dirty, then tag the page as dirty in the page cache
 * and then attach the address_space's inode to its superblock's dirty
 * inode list.
 *
 * mark_buffer_dirty() is atomic.  It takes bh->b_folio->mapping->i_private_lock,
 * i_pages lock and mapping->host->i_lock.
 */
void mark_buffer_dirty(struct buffer_head *bh)
{
	WARN_ON_ONCE(!buffer_uptodate(bh));

	trace_block_dirty_buffer(bh);

	/*
	 * Very *carefully* optimize the it-is-already-dirty case.
	 *
	 * Don't let the final "is it dirty" escape to before we
	 * perhaps modified the buffer.
	 */