From: Jan Glauber <jglauber@redhat.com> Date: Sat, 12 Jan 2008 12:39:37 +0100 Subject: [mm] make page->private usable in compound pages Message-id: 1200137977.6182.37.camel@localhost.localdomain O-Subject: [RHEL5.2 PATCH] 1/3: make page->private usable in compound pages Bugzilla: 318951 Summary: Common Code: Make page->private usable in compound pages. Description: This is a backport of two upstream patches to make page->private usable in compound (head) pages: git-commit d85f33855c303acfa87fa457157cef755b6087df Make page->private usable in compound pages git-commit 6d7779538f765963ced45a3fa4bed7ba8d2c277d mm: optimize compound_head() by avoiding a shared page flag Acked-by: Rik van Riel <riel@redhat.com> Acked-by: Larry Woodman <lwoodman@redhat.com> diff --git a/include/linux/mm.h b/include/linux/mm.h index d531eff..5d80deb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -318,17 +318,21 @@ static inline int get_page_unless_zero(struct page *page) extern void FASTCALL(__page_cache_release(struct page *)); +static inline struct page *compound_head(struct page *page) +{ + if (unlikely(PageTail(page))) + return (struct page *) page->private; + return page; +} + static inline int page_count(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); - return atomic_read(&page->_count); + return atomic_read(&compound_head(page)->_count); } static inline void get_page(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); + page = compound_head(page); atomic_inc(&page->_count); } diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index c471430..dc67b9b 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -238,6 +238,28 @@ #define __SetPageCompound(page) __set_bit(PG_compound, &(page)->flags) #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags) +/* + * PG_reclaim is used in combination with PG_compound to mark the + * head and tail of a compound page + * + * PG_compound & PG_reclaim => Tail page + * PG_compound & ~PG_reclaim => Head page + */ + +#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim)) + +#define PageTail(page) ((page->flags & PG_head_tail_mask) \ + == PG_head_tail_mask) + +#define __SetPageTail(page) ((page)->flags |= PG_head_tail_mask) + +#define __ClearPageTail(page) ((page)->flags &= ~PG_head_tail_mask) + +#define PageHead(page) ((page->flags & PG_head_tail_mask) \ + == (1L << PG_compound)) +#define __SetPageHead(page) __SetPageCompound(page) +#define __ClearPageHead(page) __ClearPageCompound(page) + #ifdef CONFIG_SWAP #define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags) #define SetPageSwapCache(page) set_bit(PG_swapcache, &(page)->flags) diff --git a/mm/internal.h b/mm/internal.h index d20e3cc..1cb8d0d 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -24,7 +24,7 @@ static inline void set_page_count(struct page *page, int v) */ static inline void set_page_refcounted(struct page *page) { - BUG_ON(PageCompound(page) && page_private(page) != (unsigned long)page); + BUG_ON(PageCompound(page) && PageTail(page)); BUG_ON(atomic_read(&page->_count)); set_page_count(page, 1); } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b7ccece..fc9b9ab 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -188,10 +188,11 @@ static void prep_compound_page(struct page *page, unsigned long order) set_compound_page_dtor(page, free_compound_page); page[1].lru.prev = (void *)order; - for (i = 0; i < nr_pages; i++) { + __SetPageHead(page); + for (i = 1; i < nr_pages; i++) { struct page *p = page + i; - __SetPageCompound(p); + __SetPageTail(p); set_page_private(p, (unsigned long)page); } } @@ -204,13 +205,14 @@ static void destroy_compound_page(struct page *page, unsigned long order) if (unlikely((unsigned long)page[1].lru.prev != order)) bad_page(page); - for (i = 0; i < nr_pages; i++) { + __ClearPageHead(page); + for (i = 1; i < nr_pages; i++) { struct page *p = page + i; - if (unlikely(!PageCompound(p) | + if (unlikely(!PageTail(p) | (page_private(p) != (unsigned long)page))) bad_page(page); - __ClearPageCompound(p); + __ClearPageTail(p); } } @@ -384,13 +386,18 @@ static inline int free_pages_check(struct page *page) 1 << PG_private | 1 << PG_locked | 1 << PG_active | - 1 << PG_reclaim | 1 << PG_slab | 1 << PG_swapcache | 1 << PG_writeback | 1 << PG_reserved | 1 << PG_buddy )))) bad_page(page); + /* + * PageReclaim == PageTail. It is only an error + * for PageReclaim to be set if PageCompound is clear. + */ + if (unlikely(!PageCompound(page) && PageReclaim(page))) + bad_page(page); if (PageDirty(page)) __ClearPageDirty(page); /* diff --git a/mm/slab.c b/mm/slab.c index 80fa86e..423fed0 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -589,8 +589,7 @@ static inline void page_set_cache(struct page *page, struct kmem_cache *cache) static inline struct kmem_cache *page_get_cache(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); + page = compound_head(page); BUG_ON(!PageSlab(page)); return (struct kmem_cache *)page->lru.next; } @@ -602,8 +601,7 @@ static inline void page_set_slab(struct page *page, struct slab *slab) static inline struct slab *page_get_slab(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); + page = compound_head(page); BUG_ON(!PageSlab(page)); return (struct slab *)page->lru.prev; } diff --git a/mm/swap.c b/mm/swap.c index 3f42431..6898238 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -45,7 +45,7 @@ int pagecache_maxpercent = 100; static void put_compound_page(struct page *page) { - page = (struct page *)page_private(page); + page = compound_head(page); if (put_page_testzero(page)) { compound_page_dtor *dtor;