gup_pgd_range

/*
 * Fast GUP
 *
 * get_user_pages_fast attempts to pin user pages by walking the page
 * tables directly and avoids taking locks. Thus the walker needs to be
 * protected from page table pages being freed from under it, and should
 * block any THP splits.
 *
 * One way to achieve this is to have the walker disable interrupts, and
 * rely on IPIs from the TLB flushing code blocking before the page table
 * pages are freed. This is unsuitable for architectures that do not need
 * to broadcast an IPI when invalidating TLBs.
 *
 * Another way to achieve this is to batch up page table containing pages
 * belonging to more than one mm_user, then rcu_sched a callback to free those
 * pages. Disabling interrupts will allow the fast_gup walker to both block
 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs
 * (which is a relatively rare event). The code below adopts this strategy.
 *
 * Before activating this code, please be aware that the following assumptions
 * are currently made:
 *
 *  *) Either HAVE_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
 *  free pages containing page tables or TLB flushing requires IPI broadcast.
 *
 *  *) ptes can be read atomically by the architecture.
 *
 *  *) access_ok is sufficient to validate userspace address ranges.
 *
 * The last two assumptions can be relaxed by the addition of helper functions.
 *
 * This code is based heavily on the PowerPC implementation by Nick Piggin.
 */
#ifdef CONFIG_HAVE_FAST_GUP
#ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
/*
 * WARNING: only to be used in the get_user_pages_fast() implementation.
 *
 * With get_user_pages_fast(), we walk down the pagetables without taking any
 * locks.  For this we would like to load the pointers atomically, but sometimes
 * that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE).  What
 * we do have is the guarantee that a PTE will only either go from not present
 * to present, or present to not present or both -- it will not switch to a
 * completely different present page without a TLB flush in between; something
 * that we are blocking by holding interrupts off.
 *
 * Setting ptes from not present to present goes:
 *
 *   ptep->pte_high = h;
 *   smp_wmb();
 *   ptep->pte_low = l;
 *
 * And present to not present goes:
 *
 *   ptep->pte_low = 0;
 *   smp_wmb();
 *   ptep->pte_high = 0;
 *
 * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
 * We load pte_high *after* loading pte_low, which ensures we don't see an older
 * value of pte_high.  *Then* we recheck pte_low, which ensures that we haven't
 * picked up a changed pte high. We might have gotten rubbish values from
 * pte_low and pte_high, but we are guaranteed that pte_low will not have the
 * present bit set *unless* it is 'l'. Because get_user_pages_fast() only
 * operates on present ptes we're safe.
 */
static inline pte_t gup_get_pte(pte_t *ptep)
{
    pte_t pte;
 
    do {
        pte.pte_low = ptep->pte_low;
        smp_rmb();
        pte.pte_high = ptep->pte_high;
        smp_rmb();
    } while (unlikely(pte.pte_low != ptep->pte_low));
 
    return pte;
}
#else /* CONFIG_GUP_GET_PTE_LOW_HIGH */
/*
 * We require that the PTE can be read atomically.
 */
static inline pte_t gup_get_pte(pte_t *ptep)
{
    return READ_ONCE(*ptep);
}
#endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
 
static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
                        struct page **pages)
{
    while ((*nr) - nr_start) {
        struct page *page = pages[--(*nr)];
 
        ClearPageReferenced(page);
        put_page(page);
    }
}
 
/*
 * Return the compund head page with ref appropriately incremented,
 * or NULL if that failed.
 */
static inline struct page *try_get_compound_head(struct page *page, int refs)
{
    struct page *head = compound_head(page);
    if (WARN_ON_ONCE(page_ref_count(head) < 0))
        return NULL;
    if (unlikely(!page_cache_add_speculative(head, refs)))
        return NULL;
    return head;
}
 
#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
             unsigned int flags, struct page **pages, int *nr)
{
    struct dev_pagemap *pgmap = NULL;
    int nr_start = *nr, ret = 0;
    pte_t *ptep, *ptem;
 
    ptem = ptep = pte_offset_map(&pmd, addr);
    do {
        pte_t pte = gup_get_pte(ptep);
        struct page *head, *page;
 
        /*
         * Similar to the PMD case below, NUMA hinting must take slow
         * path using the pte_protnone check.
         */
        if (pte_protnone(pte))
            goto pte_unmap;
 
        if (!pte_access_permitted(pte, flags & FOLL_WRITE))
            goto pte_unmap;
 
        if (pte_devmap(pte)) {
            if (unlikely(flags & FOLL_LONGTERM))
                goto pte_unmap;
 
            pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
            if (unlikely(!pgmap)) {
                undo_dev_pagemap(nr, nr_start, pages);
                goto pte_unmap;
            }
        } else if (pte_special(pte))
            goto pte_unmap;
 
        VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
        page = pte_page(pte);
 
        head = try_get_compound_head(page, 1);
        if (!head)
            goto pte_unmap;
 
        if (unlikely(pte_val(pte) != pte_val(*ptep))) {
            put_page(head);
            goto pte_unmap;
        }
 
        VM_BUG_ON_PAGE(compound_head(page) != head, page);
 
        SetPageReferenced(page);
        pages[*nr] = page;
        (*nr)++;
 
    } while (ptep++, addr += PAGE_SIZE, addr != end);
 
    ret = 1;
 
pte_unmap:
    if (pgmap)
        put_dev_pagemap(pgmap);
    pte_unmap(ptem);
    return ret;
}
#else
 
/*
 * If we can't determine whether or not a pte is special, then fail immediately
 * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
 * to be special.
 *
 * For a futex to be placed on a THP tail page, get_futex_key requires a
 * __get_user_pages_fast implementation that can pin pages. Thus it's still
 * useful to have gup_huge_pmd even if we can't operate on ptes.
 */
static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
             unsigned int flags, struct page **pages, int *nr)
{
    return 0;
}
#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
 
#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
static int __gup_device_huge(unsigned long pfn, unsigned long addr,
        unsigned long end, struct page **pages, int *nr)
{
    int nr_start = *nr;
    struct dev_pagemap *pgmap = NULL;
 
    do {
        struct page *page = pfn_to_page(pfn);
 
        pgmap = get_dev_pagemap(pfn, pgmap);
        if (unlikely(!pgmap)) {
            undo_dev_pagemap(nr, nr_start, pages);
            return 0;
        }
        SetPageReferenced(page);
        pages[*nr] = page;
        get_page(page);
        (*nr)++;
        pfn++;
    } while (addr += PAGE_SIZE, addr != end);
 
    if (pgmap)
        put_dev_pagemap(pgmap);
    return 1;
}
 
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
        unsigned long end, struct page **pages, int *nr)
{
    unsigned long fault_pfn;
    int nr_start = *nr;
 
    fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
    if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))
        return 0;
 
    if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
        undo_dev_pagemap(nr, nr_start, pages);
        return 0;
    }
    return 1;
}
 
static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
        unsigned long end, struct page **pages, int *nr)
{
    unsigned long fault_pfn;
    int nr_start = *nr;
 
    fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
    if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))
        return 0;
 
    if (unlikely(pud_val(orig) != pud_val(*pudp))) {
        undo_dev_pagemap(nr, nr_start, pages);
        return 0;
    }
    return 1;
}
#else
static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
        unsigned long end, struct page **pages, int *nr)
{
    BUILD_BUG();
    return 0;
}
 
static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
        unsigned long end, struct page **pages, int *nr)
{
    BUILD_BUG();
    return 0;
}
#endif
 
#ifdef CONFIG_ARCH_HAS_HUGEPD
static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
                      unsigned long sz)
{
    unsigned long __boundary = (addr + sz) & ~(sz-1);
    return (__boundary - 1 < end - 1) ? __boundary : end;
}
 
static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
               unsigned long end, unsigned int flags,
               struct page **pages, int *nr)
{
    unsigned long pte_end;
    struct page *head, *page;
    pte_t pte;
    int refs;
 
    pte_end = (addr + sz) & ~(sz-1);
    if (pte_end < end)
        end = pte_end;
 
    pte = READ_ONCE(*ptep);
 
    if (!pte_access_permitted(pte, flags & FOLL_WRITE))
        return 0;
 
    /* hugepages are never "special" */
    VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
 
    refs = 0;
    head = pte_page(pte);
 
    page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
    do {
        VM_BUG_ON(compound_head(page) != head);
        pages[*nr] = page;
        (*nr)++;
        page++;
        refs++;
    } while (addr += PAGE_SIZE, addr != end);
 
    head = try_get_compound_head(head, refs);
    if (!head) {
        *nr -= refs;
        return 0;
    }
 
    if (unlikely(pte_val(pte) != pte_val(*ptep))) {
        /* Could be optimized better */
        *nr -= refs;
        while (refs--)
            put_page(head);
        return 0;
    }
 
    SetPageReferenced(head);
    return 1;
}
 
static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
        unsigned int pdshift, unsigned long end, unsigned int flags,
        struct page **pages, int *nr)
{
    pte_t *ptep;
    unsigned long sz = 1UL << hugepd_shift(hugepd);
    unsigned long next;
 
    ptep = hugepte_offset(hugepd, addr, pdshift);
    do {
        next = hugepte_addr_end(addr, end, sz);
        if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
            return 0;
    } while (ptep++, addr = next, addr != end);
 
    return 1;
}
#else
static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
        unsigned int pdshift, unsigned long end, unsigned int flags,
        struct page **pages, int *nr)
{
    return 0;
}
#endif /* CONFIG_ARCH_HAS_HUGEPD */
 
static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
            unsigned long end, unsigned int flags,
            struct page **pages, int *nr)
{
    struct page *head, *page;
    int refs;
 
    if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
        return 0;
 
    if (pmd_devmap(orig)) {
        if (unlikely(flags & FOLL_LONGTERM))
            return 0;
        return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr);
    }
 
    refs = 0;
    page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
    do {
        pages[*nr] = page;
        (*nr)++;
        page++;
        refs++;
    } while (addr += PAGE_SIZE, addr != end);
 
    head = try_get_compound_head(pmd_page(orig), refs);
    if (!head) {
        *nr -= refs;
        return 0;
    }
 
    if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
        *nr -= refs;
        while (refs--)
            put_page(head);
        return 0;
    }
 
    SetPageReferenced(head);
    return 1;
}
 
static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
        unsigned long end, unsigned int flags, struct page **pages, int *nr)
{
    struct page *head, *page;
    int refs;
 
    if (!pud_access_permitted(orig, flags & FOLL_WRITE))
        return 0;
 
    if (pud_devmap(orig)) {
        if (unlikely(flags & FOLL_LONGTERM))
            return 0;
        return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr);
    }
 
    refs = 0;
    page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
    do {
        pages[*nr] = page;
        (*nr)++;
        page++;
        refs++;
    } while (addr += PAGE_SIZE, addr != end);
 
    head = try_get_compound_head(pud_page(orig), refs);
    if (!head) {
        *nr -= refs;
        return 0;
    }
 
    if (unlikely(pud_val(orig) != pud_val(*pudp))) {
        *nr -= refs;
        while (refs--)
            put_page(head);
        return 0;
    }
 
    SetPageReferenced(head);
    return 1;
}
 
static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
            unsigned long end, unsigned int flags,
            struct page **pages, int *nr)
{
    int refs;
    struct page *head, *page;
 
    if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
        return 0;
 
    BUILD_BUG_ON(pgd_devmap(orig));
    refs = 0;
    page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
    do {
        pages[*nr] = page;
        (*nr)++;
        page++;
        refs++;
    } while (addr += PAGE_SIZE, addr != end);
 
    head = try_get_compound_head(pgd_page(orig), refs);
    if (!head) {
        *nr -= refs;
        return 0;
    }
 
    if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
        *nr -= refs;
        while (refs--)
            put_page(head);
        return 0;
    }
 
    SetPageReferenced(head);
    return 1;
}
 
static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
        unsigned int flags, struct page **pages, int *nr)
{
    unsigned long next;
    pmd_t *pmdp;
 
    pmdp = pmd_offset(&pud, addr);
    do {
        pmd_t pmd = READ_ONCE(*pmdp);
 
        next = pmd_addr_end(addr, end);
        if (!pmd_present(pmd))
            return 0;
 
        if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
                 pmd_devmap(pmd))) {
            /*
             * NUMA hinting faults need to be handled in the GUP
             * slowpath for accounting purposes and so that they
             * can be serialised against THP migration.
             */
            if (pmd_protnone(pmd))
                return 0;
 
            if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
                pages, nr))
                return 0;
 
        } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
            /*
             * architecture have different format for hugetlbfs
             * pmd format and THP pmd format
             */
            if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
                     PMD_SHIFT, next, flags, pages, nr))
                return 0;
        } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
            return 0;
    } while (pmdp++, addr = next, addr != end);
 
    return 1;
}
 
static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
             unsigned int flags, struct page **pages, int *nr)
{
    unsigned long next;
    pud_t *pudp;
 
    pudp = pud_offset(&p4d, addr);
    do {
        pud_t pud = READ_ONCE(*pudp);
 
        next = pud_addr_end(addr, end);
        if (pud_none(pud))
            return 0;
        if (unlikely(pud_huge(pud))) {
            if (!gup_huge_pud(pud, pudp, addr, next, flags,
                      pages, nr))
                return 0;
        } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
            if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
                     PUD_SHIFT, next, flags, pages, nr))
                return 0;
        } else if (!gup_pmd_range(pud, addr, next, flags, pages, nr))
            return 0;
    } while (pudp++, addr = next, addr != end);
 
    return 1;
}
 
static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
             unsigned int flags, struct page **pages, int *nr)
{
    unsigned long next;
    p4d_t *p4dp;
 
    p4dp = p4d_offset(&pgd, addr);
    do {
        p4d_t p4d = READ_ONCE(*p4dp);
 
        next = p4d_addr_end(addr, end);
        if (p4d_none(p4d))
            return 0;
        BUILD_BUG_ON(p4d_huge(p4d));
        if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
            if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
                     P4D_SHIFT, next, flags, pages, nr))
                return 0;
        } else if (!gup_pud_range(p4d, addr, next, flags, pages, nr))
            return 0;
    } while (p4dp++, addr = next, addr != end);
 
    return 1;
}
 
static void gup_pgd_range(unsigned long addr, unsigned long end,
        unsigned int flags, struct page **pages, int *nr)
{
    unsigned long next;
    pgd_t *pgdp;
 
    pgdp = pgd_offset(current->mm, addr);
    do {
        pgd_t pgd = READ_ONCE(*pgdp);
 
        next = pgd_addr_end(addr, end);
        if (pgd_none(pgd))
            return;
        if (unlikely(pgd_huge(pgd))) {
            if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
                      pages, nr))
                return;
        } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
            if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
                     PGDIR_SHIFT, next, flags, pages, nr))
                return;
        } else if (!gup_p4d_range(pgd, addr, next, flags, pages, nr))
            return;
    } while (pgdp++, addr = next, addr != end);
}
#else
static inline void gup_pgd_range(unsigned long addr, unsigned long end,
        unsigned int flags, struct page **pages, int *nr)
{
}
函数源码
