[PATCH 07/11] mm: convert kmalloc_section_memmap() to populate_section_memmap()

[Dan Williams]

Allow sub-section sized ranges to be added to the memmap.
populate_section_memmap() takes an explict pfn range rather than
assuming a full section, and those parameters are plumbed all the way
through to vmmemap_populate(). There should be no sub-section in
current code. New warnings are added to clarify which memmap allocation
paths are sub-section capable.

Cc: Andrew Morton 
Cc: Mel Gorman 
Cc: Vlastimil Babka 
Cc: Johannes Weiner 
Cc: Logan Gunthorpe 
Cc: Stephen Bates 
Signed-off-by: Dan Williams 
---
 arch/x86/mm/init_64.c |4 ++-
 include/linux/mm.h|3 ++
 mm/sparse-vmemmap.c   |   24 ++--
 mm/sparse.c   |   60 -
 4 files changed, 61 insertions(+), 30 deletions(-)

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 14b9dd71d9e8..e3fb2b1be060 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1230,7 +1230,9 @@ int __meminit vmemmap_populate(unsigned long start, 
unsigned long end, int node)
struct vmem_altmap *altmap = to_vmem_altmap(start);
int err;
 
-   if (boot_cpu_has(X86_FEATURE_PSE))
+   if (end - start < PAGES_PER_SECTION * sizeof(struct page))
+   err = vmemmap_populate_basepages(start, end, node);
+   else if (boot_cpu_has(X86_FEATURE_PSE))
err = vmemmap_populate_hugepages(start, end, node, altmap);
else if (altmap) {
pr_err_once("%s: no cpu support for altmap allocations\n",
diff --git a/include/linux/mm.h b/include/linux/mm.h
index a92c8d73aeaf..7d6fb52b1f31 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2321,7 +2321,8 @@ void sparse_mem_maps_populate_node(struct page **map_map,
   unsigned long map_count,
   int nodeid);
 
-struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
+struct page *__populate_section_memmap(unsigned long pfn,
+   unsigned long nr_pages, int nid);
 pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
 pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
 pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 574c67b663fe..8679d4a81b98 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -248,20 +248,28 @@ int __meminit vmemmap_populate_basepages(unsigned long 
start,
return 0;
 }
 
-struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid)
+struct page * __meminit __populate_section_memmap(unsigned long pfn,
+   unsigned long nr_pages, int nid)
 {
unsigned long start;
unsigned long end;
-   struct page *map;
 
-   map = pfn_to_page(pnum * PAGES_PER_SECTION);
-   start = (unsigned long)map;
-   end = (unsigned long)(map + PAGES_PER_SECTION);
+   /*
+* The minimum granularity of memmap extensions is
+* SECTION_ACTIVE_SIZE as allocations are tracked in the
+* 'map_active' bitmap of the section.
+*/
+   end = ALIGN(pfn + nr_pages, PHYS_PFN(SECTION_ACTIVE_SIZE));
+   pfn &= PHYS_PFN(SECTION_ACTIVE_MASK);
+   nr_pages = end - pfn;
+
+   start = (unsigned long) pfn_to_page(pfn);
+   end = start + nr_pages * sizeof(struct page);
 
if (vmemmap_populate(start, end, nid))
return NULL;
 
-   return map;
+   return pfn_to_page(pfn);
 }
 
 void __init sparse_mem_maps_populate_node(struct page **map_map,
@@ -284,11 +292,13 @@ void __init sparse_mem_maps_populate_node(struct page 
**map_map,
 
for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
struct mem_section *ms;
+   unsigned long pfn = section_nr_to_pfn(pnum);
 
if (!present_section_nr(pnum))
continue;
 
-   map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
+   map_map[pnum] = __populate_section_memmap(pfn,
+   PAGES_PER_SECTION, nodeid);
if (map_map[pnum])
continue;
ms = __nr_to_section(pnum);
diff --git a/mm/sparse.c b/mm/sparse.c
index 00fdb5d04680..97f91770e3d0 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -423,7 +423,8 @@ static void __init sparse_early_usemaps_alloc_node(void 
*data,
 }
 
 #ifndef CONFIG_SPARSEMEM_VMEMMAP
-struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
+struct page __init *__populate_section_memmap(unsigned long pfn,
+   unsigned long nr_pages, int nid)
 {
struct page *map;
unsigned long size;
@@ -475,10 +476,12 @@ void __init sparse_mem_maps_populate_node(struct page 
**map_map,
/* fallback */
for (pnum = pnum_begin; pnum < 

[PATCH 07/11] mm: convert kmalloc_section_memmap() to populate_section_memmap()

[Dan Williams]

Allow sub-section sized ranges to be added to the memmap.
populate_section_memmap() takes an explict pfn range rather than
assuming a full section, and those parameters are plumbed all the way
through to vmmemap_populate(). There should be no sub-section in
current code. New warnings are added to clarify which memmap allocation
paths are sub-section capable.

Cc: Andrew Morton 
Cc: Mel Gorman 
Cc: Vlastimil Babka 
Cc: Johannes Weiner 
Cc: Logan Gunthorpe 
Cc: Stephen Bates 
Signed-off-by: Dan Williams 
---
 arch/x86/mm/init_64.c |4 ++-
 include/linux/mm.h|3 ++
 mm/sparse-vmemmap.c   |   24 ++--
 mm/sparse.c   |   60 -
 4 files changed, 61 insertions(+), 30 deletions(-)

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 14b9dd71d9e8..e3fb2b1be060 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1230,7 +1230,9 @@ int __meminit vmemmap_populate(unsigned long start, 
unsigned long end, int node)
struct vmem_altmap *altmap = to_vmem_altmap(start);
int err;
 
-   if (boot_cpu_has(X86_FEATURE_PSE))
+   if (end - start < PAGES_PER_SECTION * sizeof(struct page))
+   err = vmemmap_populate_basepages(start, end, node);
+   else if (boot_cpu_has(X86_FEATURE_PSE))
err = vmemmap_populate_hugepages(start, end, node, altmap);
else if (altmap) {
pr_err_once("%s: no cpu support for altmap allocations\n",
diff --git a/include/linux/mm.h b/include/linux/mm.h
index a92c8d73aeaf..7d6fb52b1f31 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2321,7 +2321,8 @@ void sparse_mem_maps_populate_node(struct page **map_map,
   unsigned long map_count,
   int nodeid);
 
-struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
+struct page *__populate_section_memmap(unsigned long pfn,
+   unsigned long nr_pages, int nid);
 pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
 pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
 pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 574c67b663fe..8679d4a81b98 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -248,20 +248,28 @@ int __meminit vmemmap_populate_basepages(unsigned long 
start,
return 0;
 }
 
-struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid)
+struct page * __meminit __populate_section_memmap(unsigned long pfn,
+   unsigned long nr_pages, int nid)
 {
unsigned long start;
unsigned long end;
-   struct page *map;
 
-   map = pfn_to_page(pnum * PAGES_PER_SECTION);
-   start = (unsigned long)map;
-   end = (unsigned long)(map + PAGES_PER_SECTION);
+   /*
+* The minimum granularity of memmap extensions is
+* SECTION_ACTIVE_SIZE as allocations are tracked in the
+* 'map_active' bitmap of the section.
+*/
+   end = ALIGN(pfn + nr_pages, PHYS_PFN(SECTION_ACTIVE_SIZE));
+   pfn &= PHYS_PFN(SECTION_ACTIVE_MASK);
+   nr_pages = end - pfn;
+
+   start = (unsigned long) pfn_to_page(pfn);
+   end = start + nr_pages * sizeof(struct page);
 
if (vmemmap_populate(start, end, nid))
return NULL;
 
-   return map;
+   return pfn_to_page(pfn);
 }
 
 void __init sparse_mem_maps_populate_node(struct page **map_map,
@@ -284,11 +292,13 @@ void __init sparse_mem_maps_populate_node(struct page 
**map_map,
 
for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
struct mem_section *ms;
+   unsigned long pfn = section_nr_to_pfn(pnum);
 
if (!present_section_nr(pnum))
continue;
 
-   map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
+   map_map[pnum] = __populate_section_memmap(pfn,
+   PAGES_PER_SECTION, nodeid);
if (map_map[pnum])
continue;
ms = __nr_to_section(pnum);
diff --git a/mm/sparse.c b/mm/sparse.c
index 00fdb5d04680..97f91770e3d0 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -423,7 +423,8 @@ static void __init sparse_early_usemaps_alloc_node(void 
*data,
 }
 
 #ifndef CONFIG_SPARSEMEM_VMEMMAP
-struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
+struct page __init *__populate_section_memmap(unsigned long pfn,
+   unsigned long nr_pages, int nid)
 {
struct page *map;
unsigned long size;
@@ -475,10 +476,12 @@ void __init sparse_mem_maps_populate_node(struct page 
**map_map,
/* fallback */
for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
struct mem_section *ms;
+   unsigned long pfn = section_nr_to_pfn(pnum);
 
if (!present_section_nr(pnum))