Address Space handling. More...

Data Structures
struct	space_callback_t

struct	space_pinned_page_t
	Pinned page structure. More...

struct	space_t
	Virtual address space structure. More...

struct	space_mapping_t
	Helper structure for managing address space mappings. More...

struct	vmm_callback_t
	Space callback structure. More...

Macros
#define	SPACE_TLB_SHOOTDOWN_TIMEOUT (CLOCKS_PER_SEC)
	The maximum time to wait for the acknowledgements from other CPU's before panicking.

Typedefs
typedef void(*	space_callback_func_t) (void *private)
	Space callback function.

Enumerations
enum	space_flags_t { SPACE_NONE = 0 , SPACE_USE_PMM_BITMAP = 1 << 0 , SPACE_MAP_KERNEL_BINARY = 1 << 1 , SPACE_MAP_KERNEL_HEAP = 1 << 2 , SPACE_MAP_IDENTITY = 1 << 3 }
	Flags for space initialization. More...

Functions
uint64_t	space_init (space_t *space, uintptr_t startAddress, uintptr_t endAddress, space_flags_t flags)
	Initializes a virtual address space.

void	space_deinit (space_t *space)
	Deinitializes a virtual address space.

void	space_load (space_t *space)
	Loads a virtual address space.

uint64_t	space_pin (space_t space, const void address, uint64_t length, stack_pointer_t *userStack)
	Pins pages within a region of the address space.

uint64_t	space_pin_terminated (space_t space, const void address, const void terminator, uint8_t objectSize, uint64_t maxCount, stack_pointer_t userStack)
	Pins a region of memory terminated by a terminator value.

void	space_unpin (space_t space, const void address, uint64_t length)
	Unpins pages in a region previously pinned with `space_pin()` or `space_pin_string()`.

uint64_t	space_check_access (space_t space, const void addr, uint64_t length)
	Checks if a virtual memory region is within the allowed address range of the space.

uint64_t	space_mapping_start (space_t space, space_mapping_t mapping, void virtAddr, void physAddr, uint64_t length, pml_flags_t flags)
	Prepare for changes to the address space mappings.

pml_callback_id_t	space_alloc_callback (space_t space, uint64_t pageAmount, space_callback_func_t func, void private)
	Allocate a callback.

void	space_free_callback (space_t *space, pml_callback_id_t callbackId)
	Free a callback.

void	space_tlb_shootdown (space_t space, void virtAddr, uint64_t pageAmount)
	Performs a TLB shootdown for a region of the address space, and wait for acknowledgements.

void *	space_mapping_end (space_t space, space_mapping_t mapping, errno_t err)
	Performs cleanup after changes to the address space mappings.

bool	space_is_mapped (space_t space, const void virtAddr, uint64_t length)
	Checks if a virtual memory region is fully mapped.

Detailed Description

Address Space handling.

Macro Definition Documentation

◆ SPACE_TLB_SHOOTDOWN_TIMEOUT

#define SPACE_TLB_SHOOTDOWN_TIMEOUT (CLOCKS_PER_SEC)

The maximum time to wait for the acknowledgements from other CPU's before panicking.

Definition at line 106 of file space.h.

Typedef Documentation

◆ space_callback_func_t

typedef void(* space_callback_func_t) (void *private)

Space callback function.

Definition at line 43 of file space.h.

Enumeration Type Documentation

◆ space_flags_t

enum space_flags_t

Flags for space initialization.

Enumerator
SPACE_NONE
SPACE_USE_PMM_BITMAP	Use the PMM bitmap to allocate the page table, this is really only for the kernel page table as it must be within a 32 bit boundary because the smp trampoline loads it as a dword.
SPACE_MAP_KERNEL_BINARY	Map the kernel binary into the address space.
SPACE_MAP_KERNEL_HEAP	Map the kernel heap into the address space.
SPACE_MAP_IDENTITY	Map the identity mapped physical memory into the address space.

Definition at line 27 of file space.h.

Function Documentation

◆ space_alloc_callback()

pml_callback_id_t space_alloc_callback	(	space_t *	space,
		uint64_t	pageAmount,
		space_callback_func_t	func,
		void *	private
	)

Allocate a callback.

Must be called between space_mapping_start() and space_mapping_end().

When pageAmount number of pages with this callback ID are unmapped or the address space is freed, the callback function will be called with the provided private data.

Parameters

space	The target address space.
pageAmount	The number of pages the callback is responsible for.
func	The callback function.
private	Private data to pass to the callback function.

Returns: On success, returns the callback ID. On failure, returns PML_MAX_CALLBACK.

Definition at line 592 of file space.c.

References bitmap_find_first_clear(), bitmap_set(), space_t::callbackBitmap, space_t::callbacks, space_t::callbacksLength, free(), space_callback_t::func, malloc(), memcpy(), memset(), NULL, space_callback_t::pageAmount, pageAmount, PML_MAX_CALLBACK, and space_callback_t::private.

Referenced by vmm_map(), and vmm_map_pages().

◆ space_check_access()

uint64_t space_check_access	(	space_t *	space,
		const void *	addr,
		uint64_t	length
	)

Checks if a virtual memory region is within the allowed address range of the space.

Checks that the given memory region is within the startAddress and endAddress range of the space, really only used in system calls that might access unmapped user space memory for example mmap(), in such cases we dont want to pin the "buffer" since we expect that it is not yet mapped.

Parameters

space	The target address space.
addr	The starting address of the memory region, can be `NULL` if length is 0.
length	The length of the memory region, in bytes.

Returns: On success, 0. On failure, ERR and errno is set.

Definition at line 474 of file space.c.

References EFAULT, EINVAL, space_t::endAddress, EOVERFLOW, ERR, errno, NULL, and space_t::startAddress.

Referenced by SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), and SYSCALL_DEFINE().

◆ space_deinit()

void space_deinit ( space_t * space )

Deinitializes a virtual address space.

Parameters

space The address space to deinitialize.

Definition at line 124 of file space.c.

References BITMAP_FOR_EACH_SET, space_t::callbackBitmap, space_t::callbacks, space_t::cpus, space_t::flags, free(), space_callback_t::func, list_is_empty(), NULL, page_table_deinit(), space_t::pageTable, panic(), space_callback_t::private, SPACE_MAP_IDENTITY, SPACE_MAP_KERNEL_BINARY, SPACE_MAP_KERNEL_HEAP, space_unmap_kernel_space_region(), VMM_IDENTITY_MAPPED_MAX, VMM_IDENTITY_MAPPED_MIN, VMM_KERNEL_BINARY_MAX, VMM_KERNEL_BINARY_MIN, VMM_KERNEL_HEAP_MAX, and VMM_KERNEL_HEAP_MIN.

Referenced by process_free(), and process_init().

◆ space_free_callback()

void space_free_callback	(	space_t *	space,
		pml_callback_id_t	callbackId
	)

Free a callback.

Must be called between space_mapping_start() and space_mapping_end().

Allows the callback ID to be reused. The callback function will not be called.

Parameters

space	The target address space.
callbackId	The callback ID to free.

Definition at line 633 of file space.c.

References bitmap_clear(), and space_t::callbackBitmap.

Referenced by vmm_map(), vmm_map_pages(), and vmm_unmap().

◆ space_init()

uint64_t space_init	(	space_t *	space,
		uintptr_t	startAddress,
		uintptr_t	endAddress,
		space_flags_t	flags
	)

Initializes a virtual address space.

Parameters

space	The address space to initialize.
startAddress	The starting address for allocations in this address space.
endAddress	The ending address for allocations in this address space.
flags	Flags to control the initialization behavior.

Returns: On success, 0. On failure, ERR and errno is set.

Definition at line 62 of file space.c.

References page_table_t::allocPages, atomic_init, BITMAP_BITS_TO_BYTES, bitmap_init(), space_t::bitmapBuffer, space_t::callbackBitmap, space_t::callbacks, space_t::callbacksLength, space_t::cpus, EINVAL, space_t::endAddress, ENOMEM, ERR, errno, space_t::flags, space_t::freeAddress, list_init(), space_t::lock, lock_init(), map_init(), memset(), NULL, page_table_init(), space_t::pageTable, space_t::pinnedPages, PML_MAX_CALLBACK, pmm_alloc_pages(), pmm_free_pages(), space_t::shootdownAcks, SPACE_MAP_IDENTITY, SPACE_MAP_KERNEL_BINARY, SPACE_MAP_KERNEL_HEAP, space_map_kernel_space_region(), space_pmm_bitmap_alloc_pages(), SPACE_USE_PMM_BITMAP, space_t::startAddress, VMM_IDENTITY_MAPPED_MAX, VMM_IDENTITY_MAPPED_MIN, VMM_KERNEL_BINARY_MAX, VMM_KERNEL_BINARY_MIN, VMM_KERNEL_HEAP_MAX, and VMM_KERNEL_HEAP_MIN.

Referenced by process_init(), and vmm_init().

◆ space_is_mapped()

bool space_is_mapped	(	space_t *	space,
		const void *	virtAddr,
		uint64_t	length
	)

Checks if a virtual memory region is fully mapped.

Parameters

space	The target address space.
virtAddr	The virtual address of the memory region.
length	The length of the memory region, in bytes.

Returns: true if the entire region is mapped, false otherwise.

Definition at line 750 of file space.c.

References BYTES_TO_PAGES, space_t::lock, LOCK_SCOPE, page_table_is_mapped(), space_t::pageTable, and space_align_region().

◆ space_load()

void space_load ( space_t * space )

Loads a virtual address space.

Must be called with interrupts disabled.

Will do nothing if the space is already loaded.

Parameters

space The address space to load.

Definition at line 161 of file space.c.

References assert, space_t::cpus, vmm_cpu_ctx_t::currentSpace, vmm_cpu_ctx_t::entry, LIST_FOR_EACH, list_push(), list_remove(), space_t::lock, lock_acquire(), lock_release(), NULL, page_table_load(), space_t::pageTable, panic(), RFLAGS_INTERRUPT_ENABLE, rflags_read(), smp_self_unsafe(), and cpu_t::vmm.

Referenced by thread_load().

◆ space_mapping_end()

void * space_mapping_end	(	space_t *	space,
		space_mapping_t *	mapping,
		errno_t	err
	)

Performs cleanup after changes to the address space mappings.

Must be called after space_mapping_start().

Parameters

space	The target address space.
mapping	The parsed information about the mapping.
err	The error code, if 0 then no error.

Returns: On success, returns the virtual address. On failure, returns NULL and errno is set.

Definition at line 730 of file space.c.

References EINVAL, EOK, errno, space_t::lock, lock_release(), NULL, space_mapping_t::pageAmount, space_update_free_address(), and space_mapping_t::virtAddr.

Referenced by vmm_alloc(), vmm_map(), vmm_map_pages(), vmm_protect(), and vmm_unmap().

◆ space_mapping_start()

uint64_t space_mapping_start	(	space_t *	space,
		space_mapping_t *	mapping,
		void *	virtAddr,
		void *	physAddr,
		uint64_t	length,
		pml_flags_t	flags
	)

Prepare for changes to the address space mappings.

Will return with the spaces mutex acquired for writing, which must be released by calling space_mapping_end().

If flags & PML_USER then the addresses must be in the user space range.

Parameters

space	The target address space.
mapping	Will be filled with parsed information about the mapping.
virtAddr	The virtual address the mapping will apply to. Can be `NULL` to let the kernel choose an address.
physAddr	The physical address to map from. Can be `NULL`.
length	The length of the virtual memory region to modify, in bytes.
flags	The page table flags for the mapping.

Returns: On success, 0. On failure, ERR.

Definition at line 524 of file space.c.

References BYTES_TO_PAGES, EFAULT, EINVAL, ENOMEM, EOVERFLOW, ERR, errno, space_mapping_t::flags, space_t::lock, lock_acquire(), lock_release(), NULL, PAGE_SIZE, space_mapping_t::pageAmount, pageAmount, space_mapping_t::physAddr, PML_ENSURE_LOWER_HALF, PML_USER, ROUND_DOWN, space_align_region(), space_find_free_region(), space_mapping_t::virtAddr, VMM_USER_SPACE_MAX, and VMM_USER_SPACE_MIN.

Referenced by vmm_alloc(), vmm_map(), vmm_map_pages(), vmm_protect(), and vmm_unmap().

◆ space_pin()

uint64_t space_pin	(	space_t *	space,
		const void *	address,
		uint64_t	length,
		stack_pointer_t *	userStack
	)

Pins pages within a region of the address space.

Used to prevent TOCTOU attacks, where a system call provides some user space region, the kernel then checks that its mapped and after that check a seperate thread in the user space process unmaps or modifies that regions mappings while the kernel is still using it.

Our solution is to pin any user space pages that are accessed or modified during the syscall, meaning that a special flag is set in the address spaces page tables that prevent those pages from being unmapped or modified until they are unpinned which happens when the syscall is finished in space_unpin().

If the region is not fully mapped, or the region is not within the spaces startAddress and endAddress range, the function will fail.

If any page in the region is already at its maximum pin depth, the calling thread will block until the page is unpinned by another thread.

If a user stack is provided and the region to pin is both unmapped and within the stack region, memory will be allocated and mapped to the relevant region in the user stack. This is needed as its possible for a user space process to pass an address to a system call that is in its user stack but not yet mapped. For example, it could create a big buffer on its stack then pass it to a syscall without first accessing it, meaning no page fault would have occurred to map the pages.

Parameters

space	The target address space.
address	The address to pin, can be `NULL` if length is 0.
length	The length of the region pointed to by `address`, in bytes.
userStack	Pointer to the user stack of the calling thread, can be `NULL, see above. @return On success,`0`. On failure,`ERR`and`errno` is set.

Definition at line 334 of file space.c.

References buffer, BYTES_TO_PAGES, EFAULT, EINVAL, EOVERFLOW, ERR, errno, space_t::lock, LOCK_SCOPE, NULL, page_table_is_mapped(), pageAmount, space_t::pageTable, space_align_region(), space_pin_depth_inc(), space_populate_user_region(), and stack_pointer_is_in_stack().

Referenced by SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), thread_copy_from_user(), thread_copy_to_user(), and thread_load_atomic_from_user().

◆ space_pin_terminated()

uint64_t space_pin_terminated	(	space_t *	space,
		const void *	address,
		const void *	terminator,
		uint8_t	objectSize,
		uint64_t	maxCount,
		stack_pointer_t *	userStack
	)

Pins a region of memory terminated by a terminator value.

Pins pages in the address space starting from address up to maxSize bytes or until the specified terminator is found.

Used for null-terminated strings or other buffers that have a specific terminator.

Parameters

space	The target address space.
address	The starting address of the region to pin.
terminator	The terminator value to search for.
objectSize	The size of each object to compare against the terminator, in bytes.
maxCount	The maximum number of objects to scan before failing.
userStack	Pointer to the user stack of the calling thread, can be `NULL`, see `space_pin()`.

Returns: On success, the number of bytes pinned, not including the terminator. On failure, ERR and errno is set.

Definition at line 382 of file space.c.

References address, EFAULT, EINVAL, ERR, errno, space_t::lock, LOCK_SCOPE, MIN, NULL, PAGE_SIZE, page_table_is_mapped(), space_t::pageTable, ROUND_UP, space_pin_depth_dec(), space_pin_depth_inc(), space_populate_user_region(), and stack_pointer_is_in_stack().

Referenced by thread_copy_from_user_pathname(), and thread_copy_from_user_terminated().

◆ space_tlb_shootdown()

void space_tlb_shootdown	(	space_t *	space,
		void *	virtAddr,
		uint64_t	pageAmount
	)

Performs a TLB shootdown for a region of the address space, and wait for acknowledgements.

Must be called between space_mapping_start() and space_mapping_end().

This will cause all CPUs that have the address space loaded to invalidate their TLB entries for the specified region.

Will not affect the current CPU's TLB, that is handled by the page_table_t directly when modifying page table entries.

TODO: Currently this does a busy wait for acknowledgements. Use a wait queue?

Parameters

space	The target address space.
virtAddr	The starting virtual address of the region.
pageAmount	The number of pages in the region.

Definition at line 638 of file space.c.

References atomic_load, atomic_store, space_t::cpus, cpu_t::id, INTERRUPT_TLB_SHOOTDOWN, lapic_send_ipi(), cpu_t::lapicId, LIST_FOR_EACH, vmm_cpu_ctx_t::lock, lock_acquire(), lock_release(), NULL, PAGE_SIZE, vmm_shootdown_t::pageAmount, pageAmount, panic(), space_t::shootdownAcks, vmm_cpu_ctx_t::shootdownCount, vmm_cpu_ctx_t::shootdowns, smp_cpu_amount(), smp_self_unsafe(), vmm_shootdown_t::space, SPACE_TLB_SHOOTDOWN_TIMEOUT, startTime, timer_uptime(), vmm_shootdown_t::virtAddr, cpu_t::vmm, and VMM_MAX_SHOOTDOWN_REQUESTS.

Referenced by vmm_page_table_unmap_with_shootdown(), and vmm_protect().

◆ space_unpin()

void space_unpin	(	space_t *	space,
		const void *	address,
		uint64_t	length
	)

Unpins pages in a region previously pinned with space_pin() or space_pin_string().

Will wake up any threads waiting to pin the same pages.

Parameters

space	The target address space.
address	The address of the region to unpin, can be `NULL` if length is 0.
length	The length of the region pointed to by `address`, in bytes.

Definition at line 448 of file space.c.

References address, BYTES_TO_PAGES, space_t::lock, LOCK_SCOPE, NULL, pageAmount, space_align_region(), and space_pin_depth_dec().

Referenced by SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), SYSCALL_DEFINE(), thread_copy_from_user(), thread_copy_from_user_pathname(), thread_copy_from_user_terminated(), thread_copy_to_user(), and thread_load_atomic_from_user().

Data Structures

Macros

Typedefs

Enumerations

Functions

Detailed Description

Macro Definition Documentation

◆ SPACE_TLB_SHOOTDOWN_TIMEOUT

Typedef Documentation

◆ space_callback_func_t

Enumeration Type Documentation

◆ space_flags_t

Function Documentation

◆ space_alloc_callback()

◆ space_check_access()

◆ space_deinit()

◆ space_free_callback()

◆ space_init()

◆ space_is_mapped()

◆ space_load()

◆ space_mapping_end()

◆ space_mapping_start()

◆ space_pin()

◆ space_pin_terminated()

◆ space_tlb_shootdown()

◆ space_unpin()