aarch64: the page tables, with 4 levels and 4 KiB pages. More...

#include <core/arch.h>
#include <core/panic.h>
#include <core/printk.h>
#include <core/string.h>
#include <core/system.h>
#include <boot/boot.h>
#include <cpu/cpu.h>
#include <sys/cpu.h>
#include <mm/pmap.h>
#include <mm/buddy.h>
#include <mm/mm.h>
#include <mm/vm.h>

Macros
#define	D_VALID (1ULL << 0)
#define	D_TABLE (1ULL << 1)
#define	D_ATTR(i)
#define	D_AP_EL0 (1ULL << 6)
#define	D_AP_RO (1ULL << 7)
#define	D_SH_INNER (3ULL << 8)
#define	D_AF (1ULL << 10)
#define	D_NG (1ULL << 11)
#define	D_PXN (1ULL << 53)
#define	D_UXN (1ULL << 54)
#define	D_ADDR(e)
#define	IDX(va, level)
#define	KERNEL_HALF(va)
#define	cur_pmap (curcpu()->arch.pmap)

Functions
static uint64_t *	table (paddr_t paddr)
	Returns a page table, through the direct map.
static paddr_t	table_alloc (void)
	Allocates an empty page table.
static void	table_free (paddr_t paddr)
	Frees a page table.
static uint64_t *	pte_get (struct pmap *pmap, vaddr_t va, int make)
	Returns the page table entry that maps va, in pmap or in the kernel's half.
void	tlb_shootdown (struct pmap *pmap, vaddr_t sva, vaddr_t eva, int leave)
void	pmap_loaded (struct pmap from, struct pmap to)
struct pmap *	pmap_switch (struct pmap *pmap)
	Switches this CPU to an address space's page tables.
static paddr_t	ram_end (void)
	Returns the end of the RAM that the boot loader found.
void	pmap_init (void)
	Takes over the boot page tables as the kernel's, at boot, and makes the page tables of the kernel's half, which every address space shares.
static void	pmap_release (struct pmap *pmap)
	Frees a pmap whose last reference went.
struct pmap *	pmap_create (void)
	Makes the page tables of a new address space: an empty user half, and the kernel's half, which every address space shares.
void	pmap_incref (struct pmap *pmap)
	Adds a reference to an address space's page tables.
void	pmap_decref (struct pmap *pmap)
	Drops a reference to an address space's page tables, freeing them with the last one.
static uint64_t	pte_flags (uint32_t flags)
	Returns the descriptor bits for a page with the VM_* flags flags.
int	pmap_add (struct pmap *pmap, vaddr_t va, paddr_t pa, uint32_t flags)
	Maps a page in an address space, replacing what was mapped there.
void	pmap_remove (struct pmap *pmap, vaddr_t sva, vaddr_t eva)
	Unmaps the pages of a range of an address space, and flushes them from the TLBs that may have them.
static void	tables_free (paddr_t paddr, int level)
	Frees a table of the user half at level and the tables under it.
void	pmap_remove_all (struct pmap *pmap)
	Unmaps the whole user half of an address space, and frees its page tables.
void	pmap_protect (struct pmap *pmap, vaddr_t sva, vaddr_t eva, uint32_t prot)
	Changes the permissions of the mapped pages of a range of an address space.
void	pmap_page_copy (paddr_t src, paddr_t dst)
	Copies a page of physical memory to another.
paddr_t	arch_page_get_mapping (struct pmap *pmap, vaddr_t vaddr)
	Finds the page mapped at a virtual address of an address space.
int	pmap_page_read (paddr_t paddr, off_t off, size_t size, void *buf)
	Reads from a page of physical memory, through the kernel's mapping.
int	pmap_page_write (paddr_t paddr, off_t off, size_t size, void *buf)
	Writes to a page of physical memory, through the kernel's mapping.
static void *	window_map (uintptr_t paddr, size_t size, uint32_t flags)
	Maps physical memory in the window, with flags.
void *	arch_phys_map (uintptr_t paddr, size_t size)
	Returns a kernel address for a range of physical memory: its address in the direct map if it's there, or a mapping made in a window.
void *	arch_mmio_map (uintptr_t paddr, size_t size)
	Maps a device's registers into the kernel's address space, uncached (memory-mapped I/O).
void	arch_mm_setup (void)
	Finishes setting up memory at boot, once mm_setup() has set up the page structures and the buddy allocator: the kernel's page tables.

Variables
struct boot *	__kboot
static struct pmap	k_pmap
static paddr_t	k_root
static paddr_t	dmap_end = ARCH_BOOT_MAPPED
static vaddr_t	mmio_next = ARCH_MMIO_BASE

Detailed Description

aarch64: the page tables, with 4 levels and 4 KiB pages.

The address space has two halves, each with its own top table. User space is TTBR0_EL1's, with one table for each pmap (pmap->map). The kernel's half is TTBR1_EL1's, with one table for all, set at boot and never switched. So the kernel's mappings are the same in every address space without copying anything, and switching address spaces only loads TTBR0_EL1.

All RAM is mapped at ARCH_DMAP_BASE, the direct map, made of the boot tables' 1 GiB blocks (boot/head.S); page tables and pages are reached there. There are no ASIDs, so a switch flushes the TLB.

Reference: the Arm Architecture Reference Manual for A-profile, D8 (The AArch64 Virtual Memory System Architecture).

Definition in file pmap.c.

Macro Definition Documentation

◆ cur_pmap

#define cur_pmap (curcpu()->arch.pmap)

Definition at line 69 of file pmap.c.

Referenced by arch_page_unmap(), pmap_init(), pmap_switch(), and tlb_shootdown().

◆ D_ADDR

#define D_ADDR ( e )

Value:

((e) & 0x0000FFFFFFFFF000ULL)

Definition at line 50 of file pmap.c.

Referenced by arch_page_get_mapping(), pmap_protect(), pmap_remove_all(), pte_get(), and tables_free().

◆ D_AF

#define D_AF (1ULL << 10)

Definition at line 45 of file pmap.c.

Referenced by pmap_init(), and pte_flags().

◆ D_AP_EL0

#define D_AP_EL0 (1ULL << 6)

Definition at line 42 of file pmap.c.

Referenced by pte_flags().

◆ D_AP_RO

#define D_AP_RO (1ULL << 7)

Definition at line 43 of file pmap.c.

Referenced by pte_flags().

◆ D_ATTR

#define D_ATTR ( i )

Value:

((uint64_t) (i) << 2)

uint64_t

unsigned long long int uint64_t

Definition stddef.h:23

Definition at line 41 of file pmap.c.

Referenced by pmap_init(), and pte_flags().

◆ D_NG

#define D_NG (1ULL << 11)

Definition at line 46 of file pmap.c.

Referenced by pte_flags().

◆ D_PXN

#define D_PXN (1ULL << 53)

Definition at line 47 of file pmap.c.

Referenced by pte_flags().

◆ D_SH_INNER

#define D_SH_INNER (3ULL << 8)

Definition at line 44 of file pmap.c.

Referenced by pmap_init(), and pte_flags().

◆ D_TABLE

#define D_TABLE (1ULL << 1)

Definition at line 40 of file pmap.c.

Referenced by pte_flags(), and pte_get().

◆ D_UXN

#define D_UXN (1ULL << 54)

Definition at line 48 of file pmap.c.

Referenced by pte_flags().

◆ D_VALID

#define D_VALID (1ULL << 0)

Definition at line 39 of file pmap.c.

Referenced by arch_page_get_mapping(), pmap_add(), pmap_init(), pmap_protect(), pmap_remove(), pmap_remove_all(), pte_flags(), pte_get(), and tables_free().

◆ IDX

#define IDX	(		va,
			level )

Value:

(((va) >> (12 + 9 * (level))) & 0x1FF)

Definition at line 55 of file pmap.c.

Referenced by pmap_init(), pte_get(), and pte_get().

◆ KERNEL_HALF

#define KERNEL_HALF ( va )

Value:

((va) >= ARCH_DMAP_BASE)

ARCH_DMAP_BASE

#define ARCH_DMAP_BASE

Definition mm.h:14

Definition at line 57 of file pmap.c.

Referenced by pmap_add(), pmap_protect(), pmap_remove(), and pte_get().

Function Documentation

◆ arch_mm_setup()

void arch_mm_setup ( void )

Finishes setting up memory at boot, once mm_setup() has set up the page structures and the buddy allocator: the kernel's page tables.

Definition at line 514 of file pmap.c.

References arch_mm_setup(), and pmap_init().

Referenced by arch_mm_setup(), and mm_setup().

◆ arch_mmio_map()

void * arch_mmio_map	(	uintptr_t	paddr,
		size_t	size )

Maps a device's registers into the kernel's address space, uncached (memory-mapped I/O).

Parameters

paddr	the registers' physical address
size	their size, in bytes

Returns: their kernel address, or NULL if they couldn't be mapped

Definition at line 508 of file pmap.c.

References arch_mmio_map(), VM_KRW, VM_NOCACHE, and window_map().

Referenced by arch_mmio_map(), e1000_probe(), pci_host_set(), platform_device_io(), x86_apic_setup(), and x86_hpet_init().

◆ arch_page_get_mapping()

paddr_t arch_page_get_mapping	(	struct pmap *	pmap,
		vaddr_t	vaddr )

Finds the page mapped at a virtual address of an address space.

Parameters

pmap	the address space's page tables
vaddr	the virtual address

Returns: the page's physical address, or 0 if none is mapped there

Definition at line 431 of file pmap.c.

References arch_page_get_mapping(), D_ADDR, D_VALID, and pte_get().

Referenced by arch_page_get_mapping(), kvmem_selftest_run(), kvmem_unmap(), mm_map(), mm_page_unmap(), and slab_page_free().

◆ arch_phys_map()

void * arch_phys_map	(	uintptr_t	paddr,
		size_t	size )

Returns a kernel address for a range of physical memory: its address in the direct map if it's there, or a mapping made in a window.

Parameters

paddr	the range's physical address
size	its size, in bytes

Returns: its kernel address, or NULL if it couldn't be mapped

Definition at line 497 of file pmap.c.

References arch_phys_map(), dmap_end, PHYS2VIRT, VM_KRW, and window_map().

Referenced by arch_phys_map(), netbuf_alloc(), page_get(), and rd_probe().

◆ pmap_add()

int pmap_add	(	struct pmap *	pmap,
		vaddr_t	va,
		paddr_t	pa,
		uint32_t	flags )

Maps a page in an address space, replacing what was mapped there.

Parameters

pmap	the page tables (an address in the kernel's half is mapped in every address space)
va	the virtual address, page-aligned
pa	the page's physical address, page-aligned
flags	the mapping's permissions and caching (VM_*)

Returns: 0, -EINVAL for an address that isn't page-aligned (or that can't be mapped page by page, in the direct map), or -ENOMEM if there was no memory for a page table

Definition at line 307 of file pmap.c.

References ARCH_DMAP_BASE, D_VALID, dmap_end, k_pmap, KERNEL_HALF, PAGE_MASK, PAGE_SIZE, pmap_add(), pte_flags(), pte_get(), and tlb_shootdown().

Referenced by kvmem_alloc(), mm_page_map(), phys_window_map(), pmap_add(), slab_page_alloc(), window_map(), and window_map().

◆ pmap_create()

struct pmap * pmap_create ( void )

Makes the page tables of a new address space: an empty user half, and the kernel's half, which every address space shares.

Returns: the new page tables, or NULL if there was no memory for them

Definition at line 242 of file pmap.c.

References kfree(), kmalloc(), M_ZERO, pmap::map, NULL, pmap_create(), pmap::ref, and table_alloc().

Referenced by pmap_create(), and proc_new().

◆ pmap_decref()

void pmap_decref ( struct pmap * pmap )

Drops a reference to an address space's page tables, freeing them with the last one.

Parameters

pmap	the page tables

Definition at line 271 of file pmap.c.

References pmap_decref(), pmap_release(), and pmap::ref.

Referenced by pmap_decref(), proc_fork(), proc_kill(), and proc_new().

◆ pmap_incref()

void pmap_incref ( struct pmap * pmap )

Adds a reference to an address space's page tables.

Parameters

pmap	the page tables

Definition at line 263 of file pmap.c.

References pmap_incref(), and pmap::ref.

Referenced by pmap_incref().

◆ pmap_init()

void pmap_init ( void )

Takes over the boot page tables as the kernel's, at boot, and makes the page tables of the kernel's half, which every address space shares.

Definition at line 195 of file pmap.c.

References ARCH_BOOT_MAPPED, cur_pmap, D_AF, D_ATTR, D_SH_INNER, D_VALID, dmap_end, k_pmap, k_root, kvm_space, LMA, pmap::map, MAX, NULL, panic, vm_space::pmap, pmap_init(), pmap_switch(), printk(), ram_end(), pmap::ref, and table_alloc().

Referenced by arch_mm_setup(), and pmap_init().

◆ pmap_loaded()

void pmap_loaded	(	struct pmap *	from,
		struct pmap *	to )

Definition at line 157 of file pmap.c.

◆ pmap_page_copy()

void pmap_page_copy	(	paddr_t	src,
		paddr_t	dst )

Copies a page of physical memory to another.

Parameters

src	the physical address of the page to copy
dst	the physical address of the page to copy it to

Definition at line 423 of file pmap.c.

References memcpy(), PAGE_SIZE, PHYS2VIRT, and pmap_page_copy().

Referenced by pf_anon(), pf_object(), pf_present(), and pmap_page_copy().

◆ pmap_page_read()

int pmap_page_read	(	paddr_t	paddr,
		off_t	off,
		size_t	size,
		void *	buf )

Reads from a page of physical memory, through the kernel's mapping.

Parameters

paddr	the page's physical address
off	the offset to read from, in the page
size	how much to read, at most what's left of the page
buf	where to put it

Returns: 0, or -EINVAL if off isn't in the page

Definition at line 440 of file pmap.c.

References memcpy(), MIN, PAGE_SIZE, PHYS2VIRT, and pmap_page_read().

Referenced by pmap_page_read().

◆ pmap_page_write()

int pmap_page_write	(	paddr_t	paddr,
		off_t	off,
		size_t	size,
		void *	buf )

Writes to a page of physical memory, through the kernel's mapping.

Parameters

paddr	the page's physical address
off	the offset to write at, in the page
size	how much to write, at most what's left of the page
buf	what to write

Returns: 0, or -EINVAL if off isn't in the page

Definition at line 452 of file pmap.c.

References memcpy(), MIN, PAGE_SIZE, PHYS2VIRT, and pmap_page_write().

Referenced by pmap_page_write(), sigcode_in(), vnode_page_in(), and vnode_pages_update().

◆ pmap_protect()

void pmap_protect	(	struct pmap *	pmap,
		vaddr_t	sva,
		vaddr_t	eva,
		uint32_t	prot )

Changes the permissions of the mapped pages of a range of an address space.

Pages that aren't mapped stay unmapped.

Parameters

pmap	the page tables
sva	the range's start, page-aligned
eva	its end
prot	the new permissions (VM_*)

Definition at line 399 of file pmap.c.

References D_ADDR, D_VALID, k_pmap, KERNEL_HALF, MIN, PAGE_MASK, PAGE_SIZE, pmap_protect(), pte_flags(), pte_get(), and tlb_shootdown().

Referenced by pf_present(), pf_zero(), pmap_protect(), and vm_space_fork().

◆ pmap_release()

void pmap_release ( struct pmap * pmap )

static

Frees a pmap whose last reference went.

Only its top table is freed: pmap_remove_all() frees the others.

Definition at line 228 of file pmap.c.

References kfree(), pmap::map, pmap_release(), table_free(), and tlb_shootdown().

Referenced by pmap_decref(), pmap_release(), pmap_release(), and pmap_release().

◆ pmap_remove()

void pmap_remove	(	struct pmap *	pmap,
		vaddr_t	sva,
		vaddr_t	eva )

Unmaps the pages of a range of an address space, and flushes them from the TLBs that may have them.

The pages lose no references.

Parameters

pmap	the page tables
sva	the range's start, page-aligned
eva	its end, page-aligned

Definition at line 337 of file pmap.c.

References D_VALID, k_pmap, KERNEL_HALF, MIN, PAGE_MASK, PAGE_SIZE, pmap_remove(), pte_get(), and tlb_shootdown().

Referenced by arch_page_unmap(), kvmem_unmap(), mm_page_unmap(), pf_zero(), phys_window_map(), pmap_remove(), slab_page_free(), window_map(), and window_map().

◆ pmap_remove_all()

void pmap_remove_all ( struct pmap * pmap )

Unmaps the whole user half of an address space, and frees its page tables.

The pages it mapped lose the reference that the page fault handler gave them (mm_page_incref()).

Parameters

pmap	the page tables

Definition at line 382 of file pmap.c.

References D_ADDR, D_VALID, pmap::map, pmap_remove_all(), table(), tables_free(), tlb_shootdown(), and USER_STACK.

Referenced by pmap_remove_all(), and vm_space_destroy().

◆ pmap_switch()

struct pmap * pmap_switch ( struct pmap * pmap )

Switches this CPU to an address space's page tables.

Parameters

pmap	the page tables to switch to

Returns: the page tables the CPU had before

Definition at line 164 of file pmap.c.

References cur_pmap, isb, pmap::map, panic, pmap_switch(), tlb_flush(), and write_sysreg.

Referenced by arch_thread_spawn(), arch_thread_switch(), init_execve(), kmain(), pmap_init(), pmap_switch(), request_do(), tlb_shootdown(), and x86_ap_start().

◆ pte_flags()

uint64_t pte_flags ( uint32_t flags )

static

Returns the descriptor bits for a page with the VM_* flags flags.

Definition at line 279 of file pmap.c.

References D_AF, D_AP_EL0, D_AP_RO, D_ATTR, D_NG, D_PXN, D_SH_INNER, D_TABLE, D_UXN, D_VALID, pte_flags(), VM_KW, VM_KX, VM_NOCACHE, VM_URWX, VM_UW, and VM_UX.

Referenced by pmap_add(), pmap_protect(), pte_flags(), pte_flags(), and pte_flags().

◆ pte_get()

uint64_t * pte_get	(	struct pmap *	pmap,
		vaddr_t	va,
		int	make )

static

Returns the page table entry that maps va, in pmap or in the kernel's half.

If make is set, missing tables on the way are made, and it returns NULL if there's no memory for one; otherwise it returns NULL if one is missing. It also returns NULL inside a block.

Definition at line 100 of file pmap.c.

References D_ADDR, D_TABLE, D_VALID, IDX, k_root, KERNEL_HALF, pmap::map, NULL, pte_get(), table(), and table_alloc().

Referenced by arch_page_get_mapping(), pmap_add(), pmap_protect(), pmap_remove(), pte_get(), pte_get(), and pte_get().

◆ ram_end()

paddr_t ram_end ( void )

static

Returns the end of the RAM that the boot loader found.

Definition at line 182 of file pmap.c.

References __kboot, mmap_t::end, boot::mmap, boot::mmap_count, MMAP_USABLE, ram_end(), and mmap_t::type.

Referenced by mm_mark_unusable(), mm_setup(), pmap_init(), ram_end(), and ram_end().

◆ table()

uint64_t * table ( paddr_t paddr )

inlinestatic

Returns a page table, through the direct map.

Definition at line 72 of file pmap.c.

References PHYS2VIRT, and table().

Referenced by arp_lookup(), arp_update(), ext2_indirect_block_get(), ext2_inode_block_read(), free_tree(), free_tree(), minix_inode_block_read(), minix_inode_block_write(), pmap_create(), pmap_init(), pmap_remove_all(), pte_get(), pte_get(), pte_get(), table(), table(), table(), table_alloc(), table_alloc(), table_alloc(), tables_free(), tables_free(), and vt_keymap().

◆ table_alloc()

paddr_t table_alloc ( void )

static

Allocates an empty page table.

Returns its physical address, or 0 if there's no memory for it.

Definition at line 79 of file pmap.c.

References memset(), mm_page_alloc(), vm_page::paddr, PAGE_SIZE, table(), and table_alloc().

Referenced by minix_inode_block_write(), pmap_create(), pmap_init(), pte_get(), pte_get(), pte_get(), table_alloc(), table_alloc(), table_alloc(), table_alloc(), table_alloc_boot(), and table_alloc_boot().

◆ table_free()

void table_free ( paddr_t paddr )

static

Frees a page table.

Definition at line 91 of file pmap.c.

References mm_page_dealloc(), vm_page::paddr, and table_free().

Referenced by pmap_release(), pmap_release(), pmap_release(), pmap_remove_all(), table_free(), table_free(), table_free(), tables_free(), and tables_free().

◆ tables_free()

void tables_free	(	paddr_t	paddr,
		int	level )

static

Frees a table of the user half at level and the tables under it.

The pages they map lose the reference that the page fault handler gave them (mm_page_incref()).

Definition at line 361 of file pmap.c.

References D_ADDR, D_VALID, mm_page_decref(), table(), table_free(), and tables_free().

Referenced by pmap_remove_all(), tables_free(), and tables_free().

◆ tlb_shootdown()

void tlb_shootdown	(	struct pmap *	pmap,
		vaddr_t	sva,
		vaddr_t	eva,
		int	leave )

Definition at line 130 of file pmap.c.

◆ window_map()

void * window_map	(	uintptr_t	paddr,
		size_t	size,
		uint32_t	flags )

static

Maps physical memory in the window, with flags.

Returns NULL if there's no room left in the window, or no memory for a page table.

Definition at line 467 of file pmap.c.

References ARCH_MMIO_BASE, ARCH_MMIO_SIZE, k_pmap, mmio_next, NULL, PAGE_ALIGN, PAGE_ROUND, PAGE_SIZE, pmap_add(), pmap_remove(), and window_map().

Referenced by arch_mmio_map(), arch_phys_map(), window_map(), and window_map().

Variable Documentation

◆ __kboot

struct boot* __kboot

extern

Definition at line 19 of file init.c.

◆ dmap_end

paddr_t dmap_end = ARCH_BOOT_MAPPED

static

Definition at line 66 of file pmap.c.

Referenced by arch_phys_map(), pmap_add(), and pmap_init().

◆ k_pmap

struct pmap k_pmap

static

Definition at line 59 of file pmap.c.

Referenced by phys_window_map(), pmap_add(), pmap_create(), pmap_init(), pmap_protect(), pmap_remove(), tlb_shootdown(), window_map(), and window_map().

◆ k_root

paddr_t k_root

static

Definition at line 62 of file pmap.c.

Referenced by pmap_init(), and pte_get().

◆ mmio_next

vaddr_t mmio_next = ARCH_MMIO_BASE

static

Definition at line 463 of file pmap.c.

Referenced by window_map(), and window_map().

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ cur_pmap

◆ D_ADDR

◆ D_AF

◆ D_AP_EL0

◆ D_AP_RO

◆ D_ATTR

◆ D_NG

◆ D_PXN

◆ D_SH_INNER

◆ D_TABLE

◆ D_UXN

◆ D_VALID

◆ IDX

◆ KERNEL_HALF

Function Documentation

◆ arch_mm_setup()

◆ arch_mmio_map()

◆ arch_page_get_mapping()

◆ arch_phys_map()

◆ pmap_add()

◆ pmap_create()

◆ pmap_decref()

◆ pmap_incref()

◆ pmap_init()

◆ pmap_loaded()

◆ pmap_page_copy()

◆ pmap_page_read()

◆ pmap_page_write()

◆ pmap_protect()

◆ pmap_release()

◆ pmap_remove()

◆ pmap_remove_all()

◆ pmap_switch()

◆ pte_flags()

◆ pte_get()

◆ ram_end()

◆ table()

◆ table_alloc()

◆ table_free()

◆ tables_free()

◆ tlb_shootdown()

◆ window_map()

Variable Documentation

◆ __kboot

◆ dmap_end

◆ k_pmap

◆ k_root

◆ mmio_next