0cad47cf13
With the recent page flag reorganisation we have a single enum which defines the valid page flags and their values, nice and clear. However there are a number of bits which are overloaded by different subsystems. Firstly there is PG_owner_priv_1 which is used by filesystems and by XEN. Secondly both SLOB and SLUB use a couple of extra page bits to manage internal state for pages they own; both overlay other bits. All of these "aliases" are scattered about the source making it very hard for a reader to know if the bits are safe to rely on in all contexts; confusion here is bad. As we now have a single place where the bits are clearly assigned it makes sense to clarify the reuse of bits by making the aliases explicit and visible with the original bit assignments. This patch creates explicit aliases within the enum itself for the overloaded bits, creates standard bit accessors PageFoo etc. and uses those throughout. This version pulls the bit manipulation out to standard named page bit accessors as suggested by Christoph, it retains the explicit mapping to the overlayed bits. A fusion of both ideas. This has been SLUB and SLOB have been compile tested on x86_64 only, and SLUB boot tested. If people feel this is worth doing then I can run a fuller set of testing. This patch: Some page flags are used for more than one purpose, for example PG_owner_priv_1. Currently there are individual accessors for each user, each built using the common flag name far away from the bit definitions. This makes it hard to see all possible uses of these bits. Now that we have a single enum to generate the bit orders it makes sense to express overlays in the same place. So create per use aliases for this bit in the main page-flags enum and use those in the accessors. [akpm@linux-foundation.org: fix xen] Signed-off-by: Andy Whitcroft <apw@shadowen.org> Cc: Pekka Enberg <penberg@cs.helsinki.fi> Cc: Christoph Lameter <cl@linux-foundation.org> Cc: Matt Mackall <mpm@selenic.com> Cc: Nick Piggin <nickpiggin@yahoo.com.au> Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Rik van Riel <riel@redhat.com> Cc: Jeremy Fitzhardinge <jeremy@goop.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
343 lines
10 KiB
C
343 lines
10 KiB
C
/*
|
|
* Macros for manipulating and testing page->flags
|
|
*/
|
|
|
|
#ifndef PAGE_FLAGS_H
|
|
#define PAGE_FLAGS_H
|
|
|
|
#include <linux/types.h>
|
|
#ifndef __GENERATING_BOUNDS_H
|
|
#include <linux/mm_types.h>
|
|
#include <linux/bounds.h>
|
|
#endif /* !__GENERATING_BOUNDS_H */
|
|
|
|
/*
|
|
* Various page->flags bits:
|
|
*
|
|
* PG_reserved is set for special pages, which can never be swapped out. Some
|
|
* of them might not even exist (eg empty_bad_page)...
|
|
*
|
|
* The PG_private bitflag is set on pagecache pages if they contain filesystem
|
|
* specific data (which is normally at page->private). It can be used by
|
|
* private allocations for its own usage.
|
|
*
|
|
* During initiation of disk I/O, PG_locked is set. This bit is set before I/O
|
|
* and cleared when writeback _starts_ or when read _completes_. PG_writeback
|
|
* is set before writeback starts and cleared when it finishes.
|
|
*
|
|
* PG_locked also pins a page in pagecache, and blocks truncation of the file
|
|
* while it is held.
|
|
*
|
|
* page_waitqueue(page) is a wait queue of all tasks waiting for the page
|
|
* to become unlocked.
|
|
*
|
|
* PG_uptodate tells whether the page's contents is valid. When a read
|
|
* completes, the page becomes uptodate, unless a disk I/O error happened.
|
|
*
|
|
* PG_referenced, PG_reclaim are used for page reclaim for anonymous and
|
|
* file-backed pagecache (see mm/vmscan.c).
|
|
*
|
|
* PG_error is set to indicate that an I/O error occurred on this page.
|
|
*
|
|
* PG_arch_1 is an architecture specific page state bit. The generic code
|
|
* guarantees that this bit is cleared for a page when it first is entered into
|
|
* the page cache.
|
|
*
|
|
* PG_highmem pages are not permanently mapped into the kernel virtual address
|
|
* space, they need to be kmapped separately for doing IO on the pages. The
|
|
* struct page (these bits with information) are always mapped into kernel
|
|
* address space...
|
|
*
|
|
* PG_buddy is set to indicate that the page is free and in the buddy system
|
|
* (see mm/page_alloc.c).
|
|
*
|
|
*/
|
|
|
|
/*
|
|
* Don't use the *_dontuse flags. Use the macros. Otherwise you'll break
|
|
* locked- and dirty-page accounting.
|
|
*
|
|
* The page flags field is split into two parts, the main flags area
|
|
* which extends from the low bits upwards, and the fields area which
|
|
* extends from the high bits downwards.
|
|
*
|
|
* | FIELD | ... | FLAGS |
|
|
* N-1 ^ 0
|
|
* (NR_PAGEFLAGS)
|
|
*
|
|
* The fields area is reserved for fields mapping zone, node (for NUMA) and
|
|
* SPARSEMEM section (for variants of SPARSEMEM that require section ids like
|
|
* SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
|
|
*/
|
|
enum pageflags {
|
|
PG_locked, /* Page is locked. Don't touch. */
|
|
PG_error,
|
|
PG_referenced,
|
|
PG_uptodate,
|
|
PG_dirty,
|
|
PG_lru,
|
|
PG_active,
|
|
PG_slab,
|
|
PG_owner_priv_1, /* Owner use. If pagecache, fs may use*/
|
|
PG_arch_1,
|
|
PG_reserved,
|
|
PG_private, /* If pagecache, has fs-private data */
|
|
PG_writeback, /* Page is under writeback */
|
|
#ifdef CONFIG_PAGEFLAGS_EXTENDED
|
|
PG_head, /* A head page */
|
|
PG_tail, /* A tail page */
|
|
#else
|
|
PG_compound, /* A compound page */
|
|
#endif
|
|
PG_swapcache, /* Swap page: swp_entry_t in private */
|
|
PG_mappedtodisk, /* Has blocks allocated on-disk */
|
|
PG_reclaim, /* To be reclaimed asap */
|
|
PG_buddy, /* Page is free, on buddy lists */
|
|
#ifdef CONFIG_IA64_UNCACHED_ALLOCATOR
|
|
PG_uncached, /* Page has been mapped as uncached */
|
|
#endif
|
|
__NR_PAGEFLAGS,
|
|
|
|
/* Filesystems */
|
|
PG_checked = PG_owner_priv_1,
|
|
|
|
/* XEN */
|
|
PG_pinned = PG_owner_priv_1,
|
|
PG_savepinned = PG_dirty,
|
|
};
|
|
|
|
#ifndef __GENERATING_BOUNDS_H
|
|
|
|
/*
|
|
* Macros to create function definitions for page flags
|
|
*/
|
|
#define TESTPAGEFLAG(uname, lname) \
|
|
static inline int Page##uname(struct page *page) \
|
|
{ return test_bit(PG_##lname, &page->flags); }
|
|
|
|
#define SETPAGEFLAG(uname, lname) \
|
|
static inline void SetPage##uname(struct page *page) \
|
|
{ set_bit(PG_##lname, &page->flags); }
|
|
|
|
#define CLEARPAGEFLAG(uname, lname) \
|
|
static inline void ClearPage##uname(struct page *page) \
|
|
{ clear_bit(PG_##lname, &page->flags); }
|
|
|
|
#define __SETPAGEFLAG(uname, lname) \
|
|
static inline void __SetPage##uname(struct page *page) \
|
|
{ __set_bit(PG_##lname, &page->flags); }
|
|
|
|
#define __CLEARPAGEFLAG(uname, lname) \
|
|
static inline void __ClearPage##uname(struct page *page) \
|
|
{ __clear_bit(PG_##lname, &page->flags); }
|
|
|
|
#define TESTSETFLAG(uname, lname) \
|
|
static inline int TestSetPage##uname(struct page *page) \
|
|
{ return test_and_set_bit(PG_##lname, &page->flags); }
|
|
|
|
#define TESTCLEARFLAG(uname, lname) \
|
|
static inline int TestClearPage##uname(struct page *page) \
|
|
{ return test_and_clear_bit(PG_##lname, &page->flags); }
|
|
|
|
|
|
#define PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \
|
|
SETPAGEFLAG(uname, lname) CLEARPAGEFLAG(uname, lname)
|
|
|
|
#define __PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \
|
|
__SETPAGEFLAG(uname, lname) __CLEARPAGEFLAG(uname, lname)
|
|
|
|
#define PAGEFLAG_FALSE(uname) \
|
|
static inline int Page##uname(struct page *page) \
|
|
{ return 0; }
|
|
|
|
#define TESTSCFLAG(uname, lname) \
|
|
TESTSETFLAG(uname, lname) TESTCLEARFLAG(uname, lname)
|
|
|
|
struct page; /* forward declaration */
|
|
|
|
PAGEFLAG(Locked, locked) TESTSCFLAG(Locked, locked)
|
|
PAGEFLAG(Error, error)
|
|
PAGEFLAG(Referenced, referenced) TESTCLEARFLAG(Referenced, referenced)
|
|
PAGEFLAG(Dirty, dirty) TESTSCFLAG(Dirty, dirty) __CLEARPAGEFLAG(Dirty, dirty)
|
|
PAGEFLAG(LRU, lru) __CLEARPAGEFLAG(LRU, lru)
|
|
PAGEFLAG(Active, active) __CLEARPAGEFLAG(Active, active)
|
|
__PAGEFLAG(Slab, slab)
|
|
PAGEFLAG(Checked, checked) /* Used by some filesystems */
|
|
PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned) /* Xen */
|
|
PAGEFLAG(SavePinned, savepinned); /* Xen */
|
|
PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved)
|
|
PAGEFLAG(Private, private) __CLEARPAGEFLAG(Private, private)
|
|
__SETPAGEFLAG(Private, private)
|
|
|
|
/*
|
|
* Only test-and-set exist for PG_writeback. The unconditional operators are
|
|
* risky: they bypass page accounting.
|
|
*/
|
|
TESTPAGEFLAG(Writeback, writeback) TESTSCFLAG(Writeback, writeback)
|
|
__PAGEFLAG(Buddy, buddy)
|
|
PAGEFLAG(MappedToDisk, mappedtodisk)
|
|
|
|
/* PG_readahead is only used for file reads; PG_reclaim is only for writes */
|
|
PAGEFLAG(Reclaim, reclaim) TESTCLEARFLAG(Reclaim, reclaim)
|
|
PAGEFLAG(Readahead, reclaim) /* Reminder to do async read-ahead */
|
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
/*
|
|
* Must use a macro here due to header dependency issues. page_zone() is not
|
|
* available at this point.
|
|
*/
|
|
#define PageHighMem(__p) is_highmem(page_zone(__p))
|
|
#else
|
|
PAGEFLAG_FALSE(HighMem)
|
|
#endif
|
|
|
|
#ifdef CONFIG_SWAP
|
|
PAGEFLAG(SwapCache, swapcache)
|
|
#else
|
|
PAGEFLAG_FALSE(SwapCache)
|
|
#endif
|
|
|
|
#ifdef CONFIG_IA64_UNCACHED_ALLOCATOR
|
|
PAGEFLAG(Uncached, uncached)
|
|
#else
|
|
PAGEFLAG_FALSE(Uncached)
|
|
#endif
|
|
|
|
static inline int PageUptodate(struct page *page)
|
|
{
|
|
int ret = test_bit(PG_uptodate, &(page)->flags);
|
|
|
|
/*
|
|
* Must ensure that the data we read out of the page is loaded
|
|
* _after_ we've loaded page->flags to check for PageUptodate.
|
|
* We can skip the barrier if the page is not uptodate, because
|
|
* we wouldn't be reading anything from it.
|
|
*
|
|
* See SetPageUptodate() for the other side of the story.
|
|
*/
|
|
if (ret)
|
|
smp_rmb();
|
|
|
|
return ret;
|
|
}
|
|
|
|
static inline void __SetPageUptodate(struct page *page)
|
|
{
|
|
smp_wmb();
|
|
__set_bit(PG_uptodate, &(page)->flags);
|
|
#ifdef CONFIG_S390
|
|
page_clear_dirty(page);
|
|
#endif
|
|
}
|
|
|
|
static inline void SetPageUptodate(struct page *page)
|
|
{
|
|
#ifdef CONFIG_S390
|
|
if (!test_and_set_bit(PG_uptodate, &page->flags))
|
|
page_clear_dirty(page);
|
|
#else
|
|
/*
|
|
* Memory barrier must be issued before setting the PG_uptodate bit,
|
|
* so that all previous stores issued in order to bring the page
|
|
* uptodate are actually visible before PageUptodate becomes true.
|
|
*
|
|
* s390 doesn't need an explicit smp_wmb here because the test and
|
|
* set bit already provides full barriers.
|
|
*/
|
|
smp_wmb();
|
|
set_bit(PG_uptodate, &(page)->flags);
|
|
#endif
|
|
}
|
|
|
|
CLEARPAGEFLAG(Uptodate, uptodate)
|
|
|
|
extern void cancel_dirty_page(struct page *page, unsigned int account_size);
|
|
|
|
int test_clear_page_writeback(struct page *page);
|
|
int test_set_page_writeback(struct page *page);
|
|
|
|
static inline void set_page_writeback(struct page *page)
|
|
{
|
|
test_set_page_writeback(page);
|
|
}
|
|
|
|
#ifdef CONFIG_PAGEFLAGS_EXTENDED
|
|
/*
|
|
* System with lots of page flags available. This allows separate
|
|
* flags for PageHead() and PageTail() checks of compound pages so that bit
|
|
* tests can be used in performance sensitive paths. PageCompound is
|
|
* generally not used in hot code paths.
|
|
*/
|
|
__PAGEFLAG(Head, head)
|
|
__PAGEFLAG(Tail, tail)
|
|
|
|
static inline int PageCompound(struct page *page)
|
|
{
|
|
return page->flags & ((1L << PG_head) | (1L << PG_tail));
|
|
|
|
}
|
|
#else
|
|
/*
|
|
* Reduce page flag use as much as possible by overlapping
|
|
* compound page flags with the flags used for page cache pages. Possible
|
|
* because PageCompound is always set for compound pages and not for
|
|
* pages on the LRU and/or pagecache.
|
|
*/
|
|
TESTPAGEFLAG(Compound, compound)
|
|
__PAGEFLAG(Head, compound)
|
|
|
|
/*
|
|
* PG_reclaim is used in combination with PG_compound to mark the
|
|
* head and tail of a compound page. This saves one page flag
|
|
* but makes it impossible to use compound pages for the page cache.
|
|
* The PG_reclaim bit would have to be used for reclaim or readahead
|
|
* if compound pages enter the page cache.
|
|
*
|
|
* PG_compound & PG_reclaim => Tail page
|
|
* PG_compound & ~PG_reclaim => Head page
|
|
*/
|
|
#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))
|
|
|
|
static inline int PageTail(struct page *page)
|
|
{
|
|
return ((page->flags & PG_head_tail_mask) == PG_head_tail_mask);
|
|
}
|
|
|
|
static inline void __SetPageTail(struct page *page)
|
|
{
|
|
page->flags |= PG_head_tail_mask;
|
|
}
|
|
|
|
static inline void __ClearPageTail(struct page *page)
|
|
{
|
|
page->flags &= ~PG_head_tail_mask;
|
|
}
|
|
|
|
#endif /* !PAGEFLAGS_EXTENDED */
|
|
|
|
#define PAGE_FLAGS (1 << PG_lru | 1 << PG_private | 1 << PG_locked | \
|
|
1 << PG_buddy | 1 << PG_writeback | \
|
|
1 << PG_slab | 1 << PG_swapcache | 1 << PG_active)
|
|
|
|
/*
|
|
* Flags checked in bad_page(). Pages on the free list should not have
|
|
* these flags set. It they are, there is a problem.
|
|
*/
|
|
#define PAGE_FLAGS_CLEAR_WHEN_BAD (PAGE_FLAGS | 1 << PG_reclaim | 1 << PG_dirty)
|
|
|
|
/*
|
|
* Flags checked when a page is freed. Pages being freed should not have
|
|
* these flags set. It they are, there is a problem.
|
|
*/
|
|
#define PAGE_FLAGS_CHECK_AT_FREE (PAGE_FLAGS | 1 << PG_reserved)
|
|
|
|
/*
|
|
* Flags checked when a page is prepped for return by the page allocator.
|
|
* Pages being prepped should not have these flags set. It they are, there
|
|
* is a problem.
|
|
*/
|
|
#define PAGE_FLAGS_CHECK_AT_PREP (PAGE_FLAGS | 1 << PG_reserved | 1 << PG_dirty)
|
|
|
|
#endif /* !__GENERATING_BOUNDS_H */
|
|
#endif /* PAGE_FLAGS_H */
|