x86_64: page tables, with 4-level paging. More...

#include <core/arch.h>
#include <core/panic.h>
#include <core/printk.h>
#include <core/string.h>
#include <core/system.h>
#include <boot/boot.h>
#include <cpu/cpu.h>
#include <sys/cpu.h>
#include <mm/pmap.h>
#include <mm/buddy.h>
#include <mm/mm.h>
#include <mm/vm.h>

Macros
#define	PG_PRESENT 0x001
#define	PG_WRITE 0x002
#define	PG_USER 0x004
#define	PG_PWT 0x008
#define	PG_PCD 0x010
#define	PG_LARGE 0x080
#define	PG_GLOBAL 0x100
#define	PG_ADDR(e)
#define	IDX(va, level)
#define	KERNEL_PML4E 256
#define	LARGE_SIZE 0x200000ULL
#define	cur_pmap (curcpu()->arch.pmap)

Functions
static uint64_t *	table (paddr_t paddr)
	Returns a page table's kernel address, in the direct map.
static paddr_t	table_alloc (void)
	Allocates an empty page table.
static paddr_t	table_alloc_boot (void)
	Allocates a page table that the kernel can't boot without: it panics if there is no memory.
static void	table_free (paddr_t paddr)
	Frees a page table.
static uint64_t *	pte_get (struct pmap *pmap, vaddr_t va, int make)
	Returns the page table entry that maps va in pmap.
struct pmap *	pmap_switch (struct pmap *pmap)
	Switches this CPU to an address space's page tables.
static paddr_t	ram_end (void)
	Returns the end of the RAM in the boot loader's memory map.
void	pmap_init (void)
	Takes over the boot page tables as the kernel's, at boot, and makes the page tables of the kernel's half, which every address space shares.
static struct pmap *	pmap_alloc (void)
	Allocates a struct pmap.
static void	pmap_release (struct pmap *pmap)
	Frees an address space's PML4, once no CPU has it loaded.
struct pmap *	pmap_create (void)
	Makes the page tables of a new address space: an empty user half, and the kernel's half, which every address space shares.
void	pmap_incref (struct pmap *pmap)
	Adds a reference to an address space's page tables.
void	pmap_decref (struct pmap *pmap)
	Drops a reference to an address space's page tables, freeing them with the last one.
static uint64_t	pte_flags (uint32_t flags)
	Returns the page table entry bits for the VM_* flags flags.
int	pmap_add (struct pmap *pmap, vaddr_t va, paddr_t pa, uint32_t flags)
	Maps a page in an address space, replacing what was mapped there.
void	pmap_remove (struct pmap *pmap, vaddr_t sva, vaddr_t eva)
	Unmaps the pages of a range of an address space, and flushes them from the TLBs that may have them.
static void	tables_free (paddr_t paddr, int level)
	Frees a page table at level and the tables under it.
void	pmap_remove_all (struct pmap *pmap)
	Unmaps the whole user half of an address space, and frees its page tables.
void	pmap_protect (struct pmap *pmap, vaddr_t sva, vaddr_t eva, uint32_t prot)
	Changes the permissions of the mapped pages of a range of an address space.
void	pmap_copy (struct pmap dst_map, struct pmap src_map, vaddr_t dst_addr, size_t len, vaddr_t src_addr)
void	pmap_update (struct pmap *pmap)
void	pmap_page_copy (paddr_t src, paddr_t dst)
	Copies a page of physical memory to another.
void	pmap_page_protect (struct vm_page *pg, uint32_t flags)
int	pmap_clear_modify (struct vm_page *pg)
int	pmap_clear_reference (struct vm_page *pg)
int	pmap_is_modified (struct vm_page *pg)
int	pmap_is_referenced (struct vm_page *pg)
paddr_t	arch_page_get_mapping (struct pmap *pmap, vaddr_t vaddr)
	Finds the page mapped at a virtual address of an address space.
int	arch_page_unmap (vaddr_t vaddr)
int	arch_mm_page_fault (vaddr_t vaddr, int err)
	Handles a page fault: turns the CPU's error code into the kernel's PF_* flags, and calls mm_page_fault() (mm/i386.c, mm/x86_64.c).
int	pmap_page_read (paddr_t paddr, off_t off, size_t size, void *buf)
	Reads from a page of physical memory, through the kernel's mapping.
int	pmap_page_write (paddr_t paddr, off_t off, size_t size, void *buf)
	Writes to a page of physical memory, through the kernel's mapping.
static void *	window_map (uintptr_t paddr, size_t size, uint32_t flags)
	Maps a range of physical memory in the window, with flags.
void *	arch_phys_map (uintptr_t paddr, size_t size)
	Returns a kernel address for a range of physical memory: its address in the direct map if it's there, or a mapping made in a window.
void *	arch_mmio_map (uintptr_t paddr, size_t size)
	Maps a device's registers into the kernel's address space, uncached (memory-mapped I/O).

Variables
struct boot *	__kboot
static struct pmap	k_pmap
static paddr_t	dmap_end
static vaddr_t	mmio_next = ARCH_MMIO_BASE

Detailed Description

x86_64: page tables, with 4-level paging.

All of physical memory is mapped at ARCH_DMAP_BASE (the direct map), and page tables and pages are reached there. Every address space has its own PML4. The PDPTs of the kernel's half (entries 256 to 511) are all made at boot and shared by every PML4, so the kernel's mappings are the same in every address space, and a PML4's kernel entries never change. Switching address spaces is loading CR3.

References: [1] - Intel(R) 64 and IA-32 Architectures Software Developer's Manual, Volume 3 (System Programming Guide), Chapter 4 (Paging), 4.5 (4-Level Paging)

Definition in file x86_64.c.

Macro Definition Documentation

◆ cur_pmap

#define cur_pmap (curcpu()->arch.pmap)

Definition at line 58 of file x86_64.c.

◆ IDX

#define IDX	(		va,
			level )

Value:

(((va) >> (12 + 9 * (level))) & 0x1FF)

Definition at line 46 of file x86_64.c.

◆ KERNEL_PML4E

#define KERNEL_PML4E 256

Definition at line 48 of file x86_64.c.

Referenced by pmap_create(), pmap_init(), pmap_remove_all(), and pte_get().

◆ LARGE_SIZE

#define LARGE_SIZE 0x200000ULL

Definition at line 49 of file x86_64.c.

Referenced by pmap_init().

◆ PG_ADDR

#define PG_ADDR ( e )

Value:

((e) & 0x000FFFFFFFFFF000ULL)

Definition at line 43 of file x86_64.c.

◆ PG_GLOBAL

#define PG_GLOBAL 0x100

Definition at line 41 of file x86_64.c.

Referenced by pmap_init().

◆ PG_LARGE

#define PG_LARGE 0x080

Definition at line 40 of file x86_64.c.

◆ PG_PCD

#define PG_PCD 0x010

Definition at line 39 of file x86_64.c.

◆ PG_PRESENT

#define PG_PRESENT 0x001

Definition at line 35 of file x86_64.c.

◆ PG_PWT

#define PG_PWT 0x008

Definition at line 38 of file x86_64.c.

◆ PG_USER

#define PG_USER 0x004

Definition at line 37 of file x86_64.c.

◆ PG_WRITE

#define PG_WRITE 0x002

Definition at line 36 of file x86_64.c.

Function Documentation

◆ arch_mm_page_fault()

int arch_mm_page_fault	(	vaddr_t	vaddr,
		int	err )

Handles a page fault: turns the CPU's error code into the kernel's PF_* flags, and calls mm_page_fault() (mm/i386.c, mm/x86_64.c).

Parameters

vaddr	the address that faulted (CR2)
err	the CPU's error code

Returns: 0 if the fault is resolved, or -EFAULT if it isn't

Definition at line 487 of file x86_64.c.

References arch_mm_page_fault(), mm_page_fault(), PF_EXEC, PF_PRESENT, PF_READ, PF_USER, and PF_WRITE.

Referenced by arch_mm_page_fault(), and isr_dispatch().

◆ arch_mmio_map()

void * arch_mmio_map	(	uintptr_t	paddr,
		size_t	size )

Maps a device's registers into the kernel's address space, uncached (memory-mapped I/O).

Parameters

paddr	the registers' physical address
size	their size, in bytes

Returns: their kernel address, or NULL if they couldn't be mapped

Definition at line 568 of file x86_64.c.

References arch_mmio_map(), VM_KRW, VM_NOCACHE, and window_map().

◆ arch_page_get_mapping()

paddr_t arch_page_get_mapping	(	struct pmap *	pmap,
		vaddr_t	vaddr )

Finds the page mapped at a virtual address of an address space.

Parameters

pmap	the address space's page tables
vaddr	the virtual address

Returns: the page's physical address, or 0 if none is mapped there

Definition at line 464 of file x86_64.c.

References arch_page_get_mapping(), PG_ADDR, PG_PRESENT, and pte_get().

◆ arch_page_unmap()

int arch_page_unmap ( vaddr_t vaddr )

Definition at line 474 of file x86_64.c.

References arch_page_unmap(), cur_pmap, PAGE_MASK, PAGE_SIZE, and pmap_remove().

◆ arch_phys_map()

void * arch_phys_map	(	uintptr_t	paddr,
		size_t	size )

Returns a kernel address for a range of physical memory: its address in the direct map if it's there, or a mapping made in a window.

Parameters

paddr	the range's physical address
size	its size, in bytes

Returns: its kernel address, or NULL if it couldn't be mapped

Definition at line 557 of file x86_64.c.

References arch_phys_map(), dmap_end, PHYS2VIRT, VM_KRW, and window_map().

◆ pmap_add()

int pmap_add	(	struct pmap *	pmap,
		vaddr_t	va,
		paddr_t	pa,
		uint32_t	flags )

Maps a page in an address space, replacing what was mapped there.

Parameters

pmap	the page tables (an address in the kernel's half is mapped in every address space)
va	the virtual address, page-aligned
pa	the page's physical address, page-aligned
flags	the mapping's permissions and caching (VM_*)

Returns: 0, -EINVAL for an address that isn't page-aligned (or that can't be mapped page by page, in the direct map), or -ENOMEM if there was no memory for a page table

Definition at line 297 of file x86_64.c.

References ARCH_DMAP_BASE, dmap_end, PAGE_MASK, PAGE_SIZE, PG_PRESENT, pmap_add(), pte_flags(), pte_get(), and tlb_shootdown().

◆ pmap_alloc()

struct pmap * pmap_alloc ( void )

static

Allocates a struct pmap.

Definition at line 212 of file x86_64.c.

References kmalloc(), and pmap_alloc().

◆ pmap_clear_modify()

int pmap_clear_modify ( struct vm_page * pg )

Definition at line 438 of file x86_64.c.

References pmap_clear_modify().

◆ pmap_clear_reference()

int pmap_clear_reference ( struct vm_page * pg )

Definition at line 444 of file x86_64.c.

References pmap_clear_reference().

◆ pmap_copy()

void pmap_copy	(	struct pmap *	dst_map,
		struct pmap *	src_map,
		vaddr_t	dst_addr,
		size_t	len,
		vaddr_t	src_addr )

Definition at line 411 of file x86_64.c.

References pmap_copy().

◆ pmap_create()

struct pmap * pmap_create ( void )

Makes the page tables of a new address space: an empty user half, and the kernel's half, which every address space shares.

Returns: the new page tables, or NULL if there was no memory for them

Definition at line 233 of file x86_64.c.

References k_pmap, KERNEL_PML4E, kfree(), pmap::map, memset(), NULL, pmap_alloc(), pmap_create(), pmap::ref, table(), and table_alloc().

◆ pmap_decref()

void pmap_decref ( struct pmap * pmap )

Drops a reference to an address space's page tables, freeing them with the last one.

Parameters

pmap	the page tables

Definition at line 270 of file x86_64.c.

References pmap_decref(), pmap_release(), and pmap::ref.

◆ pmap_incref()

void pmap_incref ( struct pmap * pmap )

Adds a reference to an address space's page tables.

Parameters

pmap	the page tables

Definition at line 261 of file x86_64.c.

References pmap_incref(), and pmap::ref.

◆ pmap_init()

void pmap_init ( void )

Takes over the boot page tables as the kernel's, at boot, and makes the page tables of the kernel's half, which every address space shares.

Definition at line 165 of file x86_64.c.

References ARCH_DMAP_BASE, cur_pmap, dmap_end, IDX, k_pmap, KERNEL_PML4E, kvm_space, LARGE_SIZE, pmap::map, NULL, PAGE_MASK, PG_ADDR, PG_GLOBAL, PG_LARGE, PG_PRESENT, PG_WRITE, vm_space::pmap, pmap_init(), pmap_loaded(), printk(), ram_end(), read_cr3(), pmap::ref, table(), table_alloc_boot(), and tlb_flush().

◆ pmap_is_modified()

int pmap_is_modified ( struct vm_page * pg )

Definition at line 450 of file x86_64.c.

References pmap_is_modified().

◆ pmap_is_referenced()

int pmap_is_referenced ( struct vm_page * pg )

Definition at line 456 of file x86_64.c.

References pmap_is_referenced().

◆ pmap_page_copy()

void pmap_page_copy	(	paddr_t	src,
		paddr_t	dst )

Copies a page of physical memory to another.

Parameters

src	the physical address of the page to copy
dst	the physical address of the page to copy it to

Definition at line 426 of file x86_64.c.

References memcpy(), PAGE_SIZE, PHYS2VIRT, and pmap_page_copy().

◆ pmap_page_protect()

void pmap_page_protect	(	struct vm_page *	pg,
		uint32_t	flags )

Definition at line 432 of file x86_64.c.

References pmap_page_protect().

◆ pmap_page_read()

int pmap_page_read	(	paddr_t	paddr,
		off_t	off,
		size_t	size,
		void *	buf )

Reads from a page of physical memory, through the kernel's mapping.

Parameters

paddr	the page's physical address
off	the offset to read from, in the page
size	how much to read, at most what's left of the page
buf	where to put it

Returns: 0, or -EINVAL if off isn't in the page

Definition at line 502 of file x86_64.c.

References memcpy(), MIN, PAGE_SIZE, PHYS2VIRT, and pmap_page_read().

◆ pmap_page_write()

int pmap_page_write	(	paddr_t	paddr,
		off_t	off,
		size_t	size,
		void *	buf )

Writes to a page of physical memory, through the kernel's mapping.

Parameters

paddr	the page's physical address
off	the offset to write at, in the page
size	how much to write, at most what's left of the page
buf	what to write

Returns: 0, or -EINVAL if off isn't in the page

Definition at line 514 of file x86_64.c.

References memcpy(), MIN, PAGE_SIZE, PHYS2VIRT, and pmap_page_write().

◆ pmap_protect()

void pmap_protect	(	struct pmap *	pmap,
		vaddr_t	sva,
		vaddr_t	eva,
		uint32_t	prot )

Changes the permissions of the mapped pages of a range of an address space.

Pages that aren't mapped stay unmapped.

Parameters

pmap	the page tables
sva	the range's start, page-aligned
eva	its end
prot	the new permissions (VM_*)

Definition at line 390 of file x86_64.c.

References MIN, PAGE_MASK, PAGE_SIZE, PG_ADDR, PG_PRESENT, pmap_protect(), pte_flags(), pte_get(), and tlb_shootdown().

◆ pmap_release()

void pmap_release ( struct pmap * pmap )

static

Frees an address space's PML4, once no CPU has it loaded.

Definition at line 218 of file x86_64.c.

References kfree(), pmap::map, pmap_release(), table_free(), and tlb_shootdown().

◆ pmap_remove()

void pmap_remove	(	struct pmap *	pmap,
		vaddr_t	sva,
		vaddr_t	eva )

Unmaps the pages of a range of an address space, and flushes them from the TLBs that may have them.

The pages lose no references.

Parameters

pmap	the page tables
sva	the range's start, page-aligned
eva	its end, page-aligned

Definition at line 326 of file x86_64.c.

References MIN, PAGE_MASK, PAGE_SIZE, PG_PRESENT, pmap_remove(), pte_get(), and tlb_shootdown().

◆ pmap_remove_all()

void pmap_remove_all ( struct pmap * pmap )

Unmaps the whole user half of an address space, and frees its page tables.

The pages it mapped lose the reference that the page fault handler gave them (mm_page_incref()).

Parameters

pmap	the page tables

Definition at line 372 of file x86_64.c.

References KERNEL_PML4E, pmap::map, PG_ADDR, PG_PRESENT, pmap_remove_all(), table(), tables_free(), tlb_shootdown(), and USER_STACK.

◆ pmap_switch()

struct pmap * pmap_switch ( struct pmap * pmap )

Switches this CPU to an address space's page tables.

Parameters

pmap	the page tables to switch to

Returns: the page tables the CPU had before

Definition at line 131 of file x86_64.c.

References cur_pmap, pmap::map, panic, pmap_loaded(), and pmap_switch().

◆ pmap_update()

void pmap_update ( struct pmap * pmap )

Definition at line 418 of file x86_64.c.

References pmap_update().

◆ pte_flags()

uint64_t pte_flags ( uint32_t flags )

static

Returns the page table entry bits for the VM_* flags flags.

Definition at line 277 of file x86_64.c.

References PG_PCD, PG_PRESENT, PG_PWT, PG_USER, PG_WRITE, pte_flags(), VM_KW, VM_NOCACHE, VM_URWX, and VM_UW.

◆ pte_get()

uint64_t * pte_get	(	struct pmap *	pmap,
		vaddr_t	va,
		int	make )

static

Returns the page table entry that maps va in pmap.

With make, the tables on the way are made if they're missing (NULL if there's no memory for one); without it, NULL if one is missing. Also NULL in the direct map's large pages.

Definition at line 101 of file x86_64.c.

References IDX, KERNEL_PML4E, pmap::map, NULL, PG_ADDR, PG_LARGE, PG_PRESENT, PG_USER, PG_WRITE, pte_get(), table(), and table_alloc().

◆ ram_end()

paddr_t ram_end ( void )

static

Returns the end of the RAM in the boot loader's memory map.

Definition at line 148 of file x86_64.c.

References __kboot, mmap_t::end, boot::mmap, boot::mmap_count, MMAP_USABLE, ram_end(), and mmap_t::type.

◆ table()

uint64_t * table ( paddr_t paddr )

inlinestatic

Returns a page table's kernel address, in the direct map.

Definition at line 61 of file x86_64.c.

References PHYS2VIRT, and table().

◆ table_alloc()

paddr_t table_alloc ( void )

static

Allocates an empty page table.

Returns its physical address, or 0 if there is no memory for it.

Definition at line 68 of file x86_64.c.

References memset(), mm_page_alloc(), vm_page::paddr, PAGE_SIZE, table(), and table_alloc().

◆ table_alloc_boot()

paddr_t table_alloc_boot ( void )

static

Allocates a page table that the kernel can't boot without: it panics if there is no memory.

Definition at line 81 of file x86_64.c.

References vm_page::paddr, panic, table_alloc(), and table_alloc_boot().

◆ table_free()

void table_free ( paddr_t paddr )

static

Frees a page table.

Definition at line 92 of file x86_64.c.

References mm_page_dealloc(), vm_page::paddr, and table_free().

◆ tables_free()

void tables_free	(	paddr_t	paddr,
		int	level )

static

Frees a page table at level and the tables under it.

The pages they map lose the reference that the page fault handler gave them (mm_page_incref()).

Definition at line 350 of file x86_64.c.

References mm_page_decref(), PG_ADDR, PG_PRESENT, table(), table_free(), and tables_free().

◆ window_map()

void * window_map	(	uintptr_t	paddr,
		size_t	size,
		uint32_t	flags )

static

Maps a range of physical memory in the window, with flags.

Returns its kernel address, or NULL if there's no room left in the window, or no memory for a page table. The window's addresses are never reused.

Definition at line 530 of file x86_64.c.

References ARCH_MMIO_BASE, ARCH_MMIO_SIZE, k_pmap, mmio_next, NULL, PAGE_ALIGN, PAGE_ROUND, PAGE_SIZE, pmap_add(), pmap_remove(), and window_map().

Variable Documentation

◆ __kboot

struct boot* __kboot

extern

Definition at line 19 of file init.c.

Referenced by aarch64_init(), arch_stack_trace(), buddy_setup(), procfs_cmdline(), procfs_rdcmdline(), ram_end(), ram_end(), rd_probe(), symbol_of(), and x86_cpu_init().

◆ dmap_end

paddr_t dmap_end

static

Definition at line 55 of file x86_64.c.

◆ k_pmap

struct pmap k_pmap

static

Definition at line 51 of file x86_64.c.

◆ mmio_next

vaddr_t mmio_next = ARCH_MMIO_BASE

static

Definition at line 525 of file x86_64.c.

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ cur_pmap

◆ IDX

◆ KERNEL_PML4E

◆ LARGE_SIZE

◆ PG_ADDR

◆ PG_GLOBAL

◆ PG_LARGE

◆ PG_PCD

◆ PG_PRESENT

◆ PG_PWT

◆ PG_USER

◆ PG_WRITE

Function Documentation

◆ arch_mm_page_fault()

◆ arch_mmio_map()

◆ arch_page_get_mapping()

◆ arch_page_unmap()

◆ arch_phys_map()

◆ pmap_add()

◆ pmap_alloc()

◆ pmap_clear_modify()

◆ pmap_clear_reference()

◆ pmap_copy()

◆ pmap_create()

◆ pmap_decref()

◆ pmap_incref()

◆ pmap_init()

◆ pmap_is_modified()

◆ pmap_is_referenced()

◆ pmap_page_copy()

◆ pmap_page_protect()

◆ pmap_page_read()

◆ pmap_page_write()

◆ pmap_protect()

◆ pmap_release()

◆ pmap_remove()

◆ pmap_remove_all()

◆ pmap_switch()

◆ pmap_update()

◆ pte_flags()

◆ pte_get()

◆ ram_end()

◆ table()

◆ table_alloc()

◆ table_alloc_boot()

◆ table_free()

◆ tables_free()

◆ window_map()

Variable Documentation

◆ __kboot

◆ dmap_end

◆ k_pmap

◆ mmio_next