From 0476e30a6d74a3500d49a4b9473a507eac665ce8 Mon Sep 17 00:00:00 2001 From: Ville Juven Date: Thu, 13 Apr 2023 16:25:22 +0300 Subject: [PATCH] mm/kmm_map: Add support to dynamically map pages into kernel virtual memory This adds functionality to map pages dynamically into kernel virtual memory. This allows implementing I/O remap for example, which is a useful (future) feature. Now, the first target is to support mapping user pages for the kernel. Why? There are some userspace structures that might be needed when the userspace process is not running. Semaphores are one such example. Signals and the WDT timeout both need access to the user semaphore to work properly. Even though for this only obtaining the kernel addressable page pool virtual address is needed, for completeness a procedure is provided to map several pages. --- arch/Kconfig | 19 ++ include/nuttx/addrenv.h | 17 ++ include/nuttx/arch.h | 102 +++++++++ include/nuttx/mm/kmap.h | 109 ++++++++++ mm/Kconfig | 10 + mm/Makefile | 1 + mm/kmap/Make.defs | 32 +++ mm/kmap/kmm_map.c | 466 ++++++++++++++++++++++++++++++++++++++++ sched/init/nx_start.c | 7 + 9 files changed, 763 insertions(+) create mode 100644 include/nuttx/mm/kmap.h create mode 100644 mm/kmap/Make.defs create mode 100644 mm/kmap/kmm_map.c diff --git a/arch/Kconfig b/arch/Kconfig index 80acae798d..f5a6b1ecb5 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -555,6 +555,13 @@ config ARCH_SHM_VBASE ---help--- The virtual address of the beginning of the shared memory region. +config ARCH_KMAP_VBASE + hex "Kernel dynamic virtual mappings base" + depends on ARCH_VMA_MAPPING + ---help--- + The virtual address of the beginning of the kernel dynamic mapping + region. + config ARCH_TEXT_NPAGES int "Max .text pages" default 1 @@ -607,6 +614,18 @@ config ARCH_SHM_NPAGES maximum number of pages per region, and the configured size of each page. +config ARCH_KMAP_NPAGES + int "Max kernel dynamic mapping pages" + default 1 + ---help--- + The maximum amount of pages that a kernel can use for dynamically + mapping physical pages to itself. + + The size of the virtual shared memory address space is then + determined by the product of the maximum number of regions, the + maximum number of pages per region, and the configured size of + each page. + endif # ARCH_VMA_MAPPING config ARCH_STACK_DYNAMIC diff --git a/include/nuttx/addrenv.h b/include/nuttx/addrenv.h index 40d8c8f110..c51216f69e 100644 --- a/include/nuttx/addrenv.h +++ b/include/nuttx/addrenv.h @@ -201,6 +201,23 @@ # define ARCH_SCRATCH_VBASE __ARCH_SHM_VBASE #endif +#ifdef CONFIG_MM_KMAP +# ifndef CONFIG_ARCH_KMAP_VBASE +# error CONFIG_ARCH_KMAP_VBASE not defined +# endif + +# if (CONFIG_ARCH_KMAP_VBASE & CONFIG_MM_MASK) != 0 +# error CONFIG_ARCH_KMAP_VBASE not aligned to page boundary +# endif + +# ifndef CONFIG_ARCH_KMAP_NPAGES +# error CONFIG_ARCH_KMAP_NPAGES not defined +# endif + +# define ARCH_KMAP_SIZE (CONFIG_ARCH_KMAP_NPAGES * CONFIG_MM_PGSIZE) +# define ARCH_KMAP_VEND (CONFIG_ARCH_KMAP_VBASE + ARCH_KMAP_SIZE - 1) +#endif + /* There is no need to use the scratch memory region if the page pool memory * is statically mapped. */ diff --git a/include/nuttx/arch.h b/include/nuttx/arch.h index 462161c10b..ee7328b7a4 100644 --- a/include/nuttx/arch.h +++ b/include/nuttx/arch.h @@ -1253,6 +1253,108 @@ int up_addrenv_kstackalloc(FAR struct tcb_s *tcb); int up_addrenv_kstackfree(FAR struct tcb_s *tcb); #endif +/**************************************************************************** + * Name: up_addrenv_find_page + * + * Description: + * Find physical page mapped to user virtual address from the address + * environment page directory. + * + * Input Parameters: + * addrenv - The user address environment. + * vaddr - The user virtual address + * + * Returned Value: + * Page physical address on success; NULL on failure. + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ADDRENV +uintptr_t up_addrenv_find_page(FAR arch_addrenv_t *addrenv, uintptr_t vaddr); +#endif + +/**************************************************************************** + * Name: up_addrenv_page_vaddr + * + * Description: + * Find the kernel virtual address associated with physical page. + * + * Input Parameters: + * page - The page physical address. + * + * Returned Value: + * Page kernel virtual address on success; NULL on failure. + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ADDRENV +uintptr_t up_addrenv_page_vaddr(uintptr_t page); +#endif + +/**************************************************************************** + * Name: up_addrenv_user_vaddr + * + * Description: + * Check if a virtual address is in user virtual address space. + * + * Input Parameters: + * vaddr - The virtual address. + * + * Returned Value: + * True if it is; false if it's not + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ADDRENV +bool up_addrenv_user_vaddr(uintptr_t vaddr); +#endif + +/**************************************************************************** + * Name: up_addrenv_kmap_pages + * + * Description: + * Map physical pages into a continuous virtual memory block. + * + * Input Parameters: + * pages - A pointer to the first element in a array of physical address, + * each corresponding to one page of memory. + * npages - The number of pages in the list of physical pages to be mapped. + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * prot - Access right flags. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +#if defined(CONFIG_ARCH_ADDRENV) && defined(CONFIG_MM_KMAP) +int up_addrenv_kmap_pages(FAR void **pages, unsigned int npages, + uintptr_t vaddr, int prot); +#endif + +/**************************************************************************** + * Name: riscv_unmap_pages + * + * Description: + * Unmap a previously mapped virtual memory region. + * + * Input Parameters: + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * npages - The number of pages to be unmapped + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +#if defined(CONFIG_ARCH_ADDRENV) && defined(CONFIG_MM_KMAP) +int up_addrenv_kunmap_pages(uintptr_t vaddr, unsigned int npages); +#endif + /**************************************************************************** * Name: up_addrenv_pa_to_va * diff --git a/include/nuttx/mm/kmap.h b/include/nuttx/mm/kmap.h new file mode 100644 index 0000000000..00a536fbe8 --- /dev/null +++ b/include/nuttx/mm/kmap.h @@ -0,0 +1,109 @@ +/**************************************************************************** + * include/nuttx/mm/kmap.h + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __INCLUDE_NUTTX_MM_KMAP_H_ +#define __INCLUDE_NUTTX_MM_KMAP_H_ + +/**************************************************************************** + * Name: kmm_map_initialize + * + * Description: + * Initialize the kernel dynamic mapping module. + * + * Input Parameters: + * None. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_map_initialize(void); + +/**************************************************************************** + * Name: kmm_map_pages + * + * Description: + * Map pages into kernel virtual memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +FAR void *kmm_map(FAR void **pages, size_t npages, int prot); + +/**************************************************************************** + * Name: kmm_unmap + * + * Description: + * Unmap a previously allocated kernel virtual memory area. + * + * Input Parameters: + * kaddr - The kernel virtual address where the mapping begins. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_unmap(FAR void *kaddr); + +/**************************************************************************** + * Name: kmm_user_map + * + * Description: + * Map a region of user memory (physical pages) for kernel use through + * a continuous virtual memory area. + * + * Input Parameters: + * uaddr - The user virtual address where mapping begins. + * size - Size of the region. + * + * Returned Value: + * Pointer to the virtual memory area, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_user_map(FAR void *uaddr, size_t size); + +/**************************************************************************** + * Name: kmm_map_user_page + * + * Description: + * Map a single physical page into kernel virtual memory. Typically just + * returns the kernel addressable page pool virtual address. + * + * Input Parameters: + * uaddr - The virtual address of the user page. + * + * Returned Value: + * Pointer to the new address environment, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_map_user_page(FAR void *uaddr); + +#endif /* __INCLUDE_NUTTX_MM_KMAP_H_ */ diff --git a/mm/Kconfig b/mm/Kconfig index 26481ba306..561ba7ed10 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -189,6 +189,16 @@ config MM_SHM Build in support for the shared memory interfaces shmget(), shmat(), shmctl(), and shmdt(). +config MM_KMAP + bool "Support for dynamic kernel virtual mappings" + default n + depends on MM_PGALLOC && BUILD_KERNEL + select ARCH_VMA_MAPPING + ---help--- + Build support for dynamically mapping pages from the page pool into + kernel virtual memory. This includes pages that are already mapped + for user. + config MM_HEAP_MEMPOOL_THRESHOLD int "The size of threshold to avoid using multiple mempool in heap" default 0 diff --git a/mm/Makefile b/mm/Makefile index a3f31de8ab..cdc72e38a1 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -34,6 +34,7 @@ include kasan/Make.defs include ubsan/Make.defs include tlsf/Make.defs include map/Make.defs +include kmap/Make.defs BINDIR ?= bin diff --git a/mm/kmap/Make.defs b/mm/kmap/Make.defs new file mode 100644 index 0000000000..993d45877d --- /dev/null +++ b/mm/kmap/Make.defs @@ -0,0 +1,32 @@ +############################################################################ +# mm/kmap/Make.defs +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +ifeq ($(CONFIG_MM_KMAP),y) + +# Kernel virtual mapping support + +CSRCS += kmm_map.c + +# Add the shared memory directory to the build + +DEPPATH += --dep-path kmap +VPATH += :kmap + +endif diff --git a/mm/kmap/kmm_map.c b/mm/kmap/kmm_map.c new file mode 100644 index 0000000000..5efdfe5074 --- /dev/null +++ b/mm/kmap/kmm_map.c @@ -0,0 +1,466 @@ +/**************************************************************************** + * mm/kmap/kmm_map.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if defined(CONFIG_BUILD_KERNEL) + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +static GRAN_HANDLE g_kmm_map_vpages; +static struct mm_map_s g_kmm_map; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: get_user_pages + * + * Description: + * Get the physical pages mapped to a user virtual memory region. + * + * Input Parameters: + * pages - Pointer to buffer where the page addresses are recorded. + * npages - Amount of pages. + * vaddr - Start address of the user virtual memory region. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +static int get_user_pages(FAR void **pages, size_t npages, uintptr_t vaddr) +{ + FAR struct tcb_s *tcb = nxsched_self(); + uintptr_t page; + int i; + + /* Find the pages associated with the user virtual address space */ + + for (i = 0; i < npages; i++, vaddr += MM_PGSIZE) + { + page = up_addrenv_find_page(&tcb->addrenv_own->addrenv, vaddr); + if (!page) + { + /* Something went wrong, get out */ + + return -EINVAL; + } + + pages[i] = (FAR void *)page; + } + + return OK; +} + +/**************************************************************************** + * Name: map_pages + * + * Description: + * Map pages into kernel virtual memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +static void *map_pages(FAR void **pages, size_t npages, int prot) +{ + struct mm_map_entry_s entry; + FAR void *vaddr; + size_t size; + int ret; + + /* The region size is full pages */ + + size = npages << MM_PGSHIFT; + + /* Find a virtual memory area that fits */ + + vaddr = gran_alloc(&g_kmm_map_vpages, size); + if (!vaddr) + { + return NULL; + } + + /* Map the pages into the kernel page directory */ + + ret = up_addrenv_kmap_pages(pages, npages, (uintptr_t)vaddr, prot); + if (ret < 0) + { + goto errout_with_vaddr; + } + + entry.vaddr = vaddr; + entry.length = size; + entry.offset = 0; + entry.munmap = NULL; + + ret = mm_map_add(&g_kmm_map, &entry); + if (ret < 0) + { + goto errout_with_pgmap; + } + + return vaddr; + +errout_with_pgmap: + up_addrenv_kunmap_pages((uintptr_t)vaddr, npages); +errout_with_vaddr: + gran_free(&g_kmm_map_vpages, vaddr, size); + return NULL; +} + +/**************************************************************************** + * Name: map_single_user_page + * + * Description: + * Map a single user page into kernel memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +static void *map_single_user_page(uintptr_t vaddr) +{ + FAR struct tcb_s *tcb = nxsched_self(); + uintptr_t page; + + /* Find the page associated with this virtual address */ + + page = up_addrenv_find_page(&tcb->addrenv_own->addrenv, vaddr); + if (!page) + { + return NULL; + } + + vaddr = up_addrenv_page_vaddr(page); + return (FAR void *)vaddr; +} + +/**************************************************************************** + * Name: kmm_map_lock + * + * Description: + * Get exclusive access to the kernel mm_map + * + ****************************************************************************/ + +static int kmm_map_lock(void) +{ + return nxrmutex_lock(&g_kmm_map.mm_map_mutex); +} + +/**************************************************************************** + * Name: kmm_map_unlock + * + * Description: + * Relinquish exclusive access to the kernel mm_map + * + ****************************************************************************/ + +static void kmm_map_unlock(void) +{ + DEBUGVERIFY(nxrmutex_unlock(&g_kmm_map.mm_map_mutex)); +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: kmm_map_initialize + * + * Description: + * Initialize the kernel dynamic mapping module. + * + * Input Parameters: + * None. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_map_initialize(void) +{ + g_kmm_map_vpages = gran_initialize((FAR void *)CONFIG_ARCH_KMAP_VBASE, + CONFIG_ARCH_KMAP_NPAGES << MM_PGSHIFT, + MM_PGSHIFT, MM_PGSHIFT); + DEBUGVERIFY(g_kmm_map_vpages); + mm_map_initialize(&g_kmm_map, true); +} + +/**************************************************************************** + * Name: kmm_map_pages + * + * Description: + * Map pages into kernel virtual memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +FAR void *kmm_map(FAR void **pages, size_t npages, int prot) +{ + uintptr_t vaddr; + + if (!pages || !npages || npages > CONFIG_ARCH_KMAP_NPAGES) + { + return NULL; + } + + /* Attempt to map the pages */ + + vaddr = (uintptr_t)map_pages(pages, npages, prot); + if (!vaddr) + { + return NULL; + } + + return (FAR void *)(vaddr); +} + +/**************************************************************************** + * Name: kmm_unmap + * + * Description: + * Unmap a previously allocated kernel virtual memory area. + * + * Input Parameters: + * kaddr - The kernel virtual address where the mapping begins. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_unmap(FAR void *kaddr) +{ + FAR struct mm_map_entry_s *entry; + unsigned int npages; + int ret; + + /* Lock the mapping list when we fiddle around with it */ + + ret = kmm_map_lock(); + if (ret == OK) + { + /* Find the entry, it is OK if none found */ + + entry = mm_map_find(get_current_mm(), kaddr, 1); + if (entry) + { + npages = MM_NPAGES(entry->length); + + /* Remove the mappings from the page directory */ + + up_addrenv_kunmap_pages((uintptr_t)entry->vaddr, npages); + + /* Release the virtual memory area for use */ + + gran_free(&g_kmm_map_vpages, entry->vaddr, entry->length); + + /* Remove the mapping from the kernel mapping list */ + + mm_map_remove(&g_kmm_map, entry); + } + + kmm_map_unlock(); + } +} + +/**************************************************************************** + * Name: kmm_user_map + * + * Description: + * Map a region of user memory (physical pages) for kernel use through + * a continuous virtual memory area. + * + * Input Parameters: + * uaddr - The user virtual address where mapping begins. + * size - Size of the region. + * + * Returned Value: + * Pointer to the virtual memory area, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_user_map(FAR void *uaddr, size_t size) +{ + FAR void *pages; + uintptr_t vaddr; + uintptr_t offset; + size_t npages; + int ret; + + /* Find the kernel addressable virtual address, or map the user memory */ + + vaddr = (uintptr_t)uaddr; + + /* If the memory is not user memory, get out */ + + if (!up_addrenv_user_vaddr(vaddr)) + { + /* Not user memory, get out */ + + return uaddr; + } + + /* How many pages (including partial) does the space encompass ? */ + + offset = vaddr & MM_PGMASK; + vaddr = MM_PGALIGNDOWN(vaddr); + npages = MM_NPAGES(offset + size); + + /* Does the area fit in 1 page, including page boundary crossings ? */ + + if (npages == 1) + { + /* Yes, can simply return the kernel addressable virtual address */ + + vaddr = (uintptr_t)map_single_user_page(vaddr); + return (FAR void *)(vaddr + offset); + } + + /* No, the area must be mapped into kernel virtual address space */ + + pages = kmm_zalloc(npages * sizeof(void *)); + if (!pages) + { + return NULL; + } + + /* Fetch the physical pages for the user virtual address range */ + + ret = get_user_pages(&pages, npages, vaddr); + if (ret < 0) + { + goto errout_with_pages; + } + + /* Map the physical pages to kernel memory */ + + vaddr = (uintptr_t)map_pages(&pages, npages, PROT_READ | PROT_WRITE); + if (!vaddr) + { + goto errout_with_pages; + } + + /* Ok, we have a virtual memory area, add the offset back */ + + return (FAR void *)(vaddr + offset); + +errout_with_pages: + kmm_free(pages); + return NULL; +} + +/**************************************************************************** + * Name: kmm_map_user_page + * + * Description: + * Map a single physical page into kernel virtual memory. Typically just + * returns the kernel addressable page pool virtual address. + * + * Input Parameters: + * uaddr - The virtual address of the user page. + * + * Returned Value: + * Pointer to the new address environment, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_map_user_page(FAR void *uaddr) +{ + uintptr_t vaddr; + uintptr_t offset; + + /* Find the kernel addressable virtual address, or map the user memory */ + + vaddr = (uintptr_t)uaddr; + + /* If the memory is not user memory, get out */ + + if (!up_addrenv_user_vaddr(vaddr)) + { + /* Not user memory, get out */ + + return uaddr; + } + + /* Record the offset and add it back later */ + + offset = vaddr & MM_PGMASK; + vaddr = MM_PGALIGNDOWN(vaddr); + + vaddr = (uintptr_t)map_single_user_page(vaddr); + if (!vaddr) + { + return NULL; + } + + return (FAR void *)(vaddr + offset); +} + +#endif diff --git a/sched/init/nx_start.c b/sched/init/nx_start.c index df74373a0a..9bb37f965c 100644 --- a/sched/init/nx_start.c +++ b/sched/init/nx_start.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -472,6 +473,12 @@ void nx_start(void) } #endif +#ifdef CONFIG_MM_KMAP + /* Initialize the kernel dynamic mapping module */ + + kmm_map_initialize(); +#endif + #ifdef CONFIG_ARCH_HAVE_EXTRA_HEAPS /* Initialize any extra heap. */