diff --git a/arch/Kconfig b/arch/Kconfig index 80acae798d..f5a6b1ecb5 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -555,6 +555,13 @@ config ARCH_SHM_VBASE ---help--- The virtual address of the beginning of the shared memory region. +config ARCH_KMAP_VBASE + hex "Kernel dynamic virtual mappings base" + depends on ARCH_VMA_MAPPING + ---help--- + The virtual address of the beginning of the kernel dynamic mapping + region. + config ARCH_TEXT_NPAGES int "Max .text pages" default 1 @@ -607,6 +614,18 @@ config ARCH_SHM_NPAGES maximum number of pages per region, and the configured size of each page. +config ARCH_KMAP_NPAGES + int "Max kernel dynamic mapping pages" + default 1 + ---help--- + The maximum amount of pages that a kernel can use for dynamically + mapping physical pages to itself. + + The size of the virtual shared memory address space is then + determined by the product of the maximum number of regions, the + maximum number of pages per region, and the configured size of + each page. + endif # ARCH_VMA_MAPPING config ARCH_STACK_DYNAMIC diff --git a/include/nuttx/addrenv.h b/include/nuttx/addrenv.h index 40d8c8f110..c51216f69e 100644 --- a/include/nuttx/addrenv.h +++ b/include/nuttx/addrenv.h @@ -201,6 +201,23 @@ # define ARCH_SCRATCH_VBASE __ARCH_SHM_VBASE #endif +#ifdef CONFIG_MM_KMAP +# ifndef CONFIG_ARCH_KMAP_VBASE +# error CONFIG_ARCH_KMAP_VBASE not defined +# endif + +# if (CONFIG_ARCH_KMAP_VBASE & CONFIG_MM_MASK) != 0 +# error CONFIG_ARCH_KMAP_VBASE not aligned to page boundary +# endif + +# ifndef CONFIG_ARCH_KMAP_NPAGES +# error CONFIG_ARCH_KMAP_NPAGES not defined +# endif + +# define ARCH_KMAP_SIZE (CONFIG_ARCH_KMAP_NPAGES * CONFIG_MM_PGSIZE) +# define ARCH_KMAP_VEND (CONFIG_ARCH_KMAP_VBASE + ARCH_KMAP_SIZE - 1) +#endif + /* There is no need to use the scratch memory region if the page pool memory * is statically mapped. */ diff --git a/include/nuttx/arch.h b/include/nuttx/arch.h index 462161c10b..ee7328b7a4 100644 --- a/include/nuttx/arch.h +++ b/include/nuttx/arch.h @@ -1253,6 +1253,108 @@ int up_addrenv_kstackalloc(FAR struct tcb_s *tcb); int up_addrenv_kstackfree(FAR struct tcb_s *tcb); #endif +/**************************************************************************** + * Name: up_addrenv_find_page + * + * Description: + * Find physical page mapped to user virtual address from the address + * environment page directory. + * + * Input Parameters: + * addrenv - The user address environment. + * vaddr - The user virtual address + * + * Returned Value: + * Page physical address on success; NULL on failure. + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ADDRENV +uintptr_t up_addrenv_find_page(FAR arch_addrenv_t *addrenv, uintptr_t vaddr); +#endif + +/**************************************************************************** + * Name: up_addrenv_page_vaddr + * + * Description: + * Find the kernel virtual address associated with physical page. + * + * Input Parameters: + * page - The page physical address. + * + * Returned Value: + * Page kernel virtual address on success; NULL on failure. + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ADDRENV +uintptr_t up_addrenv_page_vaddr(uintptr_t page); +#endif + +/**************************************************************************** + * Name: up_addrenv_user_vaddr + * + * Description: + * Check if a virtual address is in user virtual address space. + * + * Input Parameters: + * vaddr - The virtual address. + * + * Returned Value: + * True if it is; false if it's not + * + ****************************************************************************/ + +#ifdef CONFIG_ARCH_ADDRENV +bool up_addrenv_user_vaddr(uintptr_t vaddr); +#endif + +/**************************************************************************** + * Name: up_addrenv_kmap_pages + * + * Description: + * Map physical pages into a continuous virtual memory block. + * + * Input Parameters: + * pages - A pointer to the first element in a array of physical address, + * each corresponding to one page of memory. + * npages - The number of pages in the list of physical pages to be mapped. + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * prot - Access right flags. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +#if defined(CONFIG_ARCH_ADDRENV) && defined(CONFIG_MM_KMAP) +int up_addrenv_kmap_pages(FAR void **pages, unsigned int npages, + uintptr_t vaddr, int prot); +#endif + +/**************************************************************************** + * Name: riscv_unmap_pages + * + * Description: + * Unmap a previously mapped virtual memory region. + * + * Input Parameters: + * vaddr - The virtual address corresponding to the beginning of the + * (continuous) virtual address region. + * npages - The number of pages to be unmapped + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +#if defined(CONFIG_ARCH_ADDRENV) && defined(CONFIG_MM_KMAP) +int up_addrenv_kunmap_pages(uintptr_t vaddr, unsigned int npages); +#endif + /**************************************************************************** * Name: up_addrenv_pa_to_va * diff --git a/include/nuttx/mm/kmap.h b/include/nuttx/mm/kmap.h new file mode 100644 index 0000000000..00a536fbe8 --- /dev/null +++ b/include/nuttx/mm/kmap.h @@ -0,0 +1,109 @@ +/**************************************************************************** + * include/nuttx/mm/kmap.h + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +#ifndef __INCLUDE_NUTTX_MM_KMAP_H_ +#define __INCLUDE_NUTTX_MM_KMAP_H_ + +/**************************************************************************** + * Name: kmm_map_initialize + * + * Description: + * Initialize the kernel dynamic mapping module. + * + * Input Parameters: + * None. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_map_initialize(void); + +/**************************************************************************** + * Name: kmm_map_pages + * + * Description: + * Map pages into kernel virtual memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +FAR void *kmm_map(FAR void **pages, size_t npages, int prot); + +/**************************************************************************** + * Name: kmm_unmap + * + * Description: + * Unmap a previously allocated kernel virtual memory area. + * + * Input Parameters: + * kaddr - The kernel virtual address where the mapping begins. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_unmap(FAR void *kaddr); + +/**************************************************************************** + * Name: kmm_user_map + * + * Description: + * Map a region of user memory (physical pages) for kernel use through + * a continuous virtual memory area. + * + * Input Parameters: + * uaddr - The user virtual address where mapping begins. + * size - Size of the region. + * + * Returned Value: + * Pointer to the virtual memory area, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_user_map(FAR void *uaddr, size_t size); + +/**************************************************************************** + * Name: kmm_map_user_page + * + * Description: + * Map a single physical page into kernel virtual memory. Typically just + * returns the kernel addressable page pool virtual address. + * + * Input Parameters: + * uaddr - The virtual address of the user page. + * + * Returned Value: + * Pointer to the new address environment, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_map_user_page(FAR void *uaddr); + +#endif /* __INCLUDE_NUTTX_MM_KMAP_H_ */ diff --git a/mm/Kconfig b/mm/Kconfig index 26481ba306..561ba7ed10 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -189,6 +189,16 @@ config MM_SHM Build in support for the shared memory interfaces shmget(), shmat(), shmctl(), and shmdt(). +config MM_KMAP + bool "Support for dynamic kernel virtual mappings" + default n + depends on MM_PGALLOC && BUILD_KERNEL + select ARCH_VMA_MAPPING + ---help--- + Build support for dynamically mapping pages from the page pool into + kernel virtual memory. This includes pages that are already mapped + for user. + config MM_HEAP_MEMPOOL_THRESHOLD int "The size of threshold to avoid using multiple mempool in heap" default 0 diff --git a/mm/Makefile b/mm/Makefile index a3f31de8ab..cdc72e38a1 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -34,6 +34,7 @@ include kasan/Make.defs include ubsan/Make.defs include tlsf/Make.defs include map/Make.defs +include kmap/Make.defs BINDIR ?= bin diff --git a/mm/kmap/Make.defs b/mm/kmap/Make.defs new file mode 100644 index 0000000000..993d45877d --- /dev/null +++ b/mm/kmap/Make.defs @@ -0,0 +1,32 @@ +############################################################################ +# mm/kmap/Make.defs +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. The +# ASF licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +############################################################################ + +ifeq ($(CONFIG_MM_KMAP),y) + +# Kernel virtual mapping support + +CSRCS += kmm_map.c + +# Add the shared memory directory to the build + +DEPPATH += --dep-path kmap +VPATH += :kmap + +endif diff --git a/mm/kmap/kmm_map.c b/mm/kmap/kmm_map.c new file mode 100644 index 0000000000..5efdfe5074 --- /dev/null +++ b/mm/kmap/kmm_map.c @@ -0,0 +1,466 @@ +/**************************************************************************** + * mm/kmap/kmm_map.c + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The + * ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + ****************************************************************************/ + +/**************************************************************************** + * Included Files + ****************************************************************************/ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if defined(CONFIG_BUILD_KERNEL) + +/**************************************************************************** + * Pre-processor Definitions + ****************************************************************************/ + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +static GRAN_HANDLE g_kmm_map_vpages; +static struct mm_map_s g_kmm_map; + +/**************************************************************************** + * Private Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: get_user_pages + * + * Description: + * Get the physical pages mapped to a user virtual memory region. + * + * Input Parameters: + * pages - Pointer to buffer where the page addresses are recorded. + * npages - Amount of pages. + * vaddr - Start address of the user virtual memory region. + * + * Returned Value: + * Zero (OK) is returned on success; a negated errno value is returned + * on failure. + * + ****************************************************************************/ + +static int get_user_pages(FAR void **pages, size_t npages, uintptr_t vaddr) +{ + FAR struct tcb_s *tcb = nxsched_self(); + uintptr_t page; + int i; + + /* Find the pages associated with the user virtual address space */ + + for (i = 0; i < npages; i++, vaddr += MM_PGSIZE) + { + page = up_addrenv_find_page(&tcb->addrenv_own->addrenv, vaddr); + if (!page) + { + /* Something went wrong, get out */ + + return -EINVAL; + } + + pages[i] = (FAR void *)page; + } + + return OK; +} + +/**************************************************************************** + * Name: map_pages + * + * Description: + * Map pages into kernel virtual memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +static void *map_pages(FAR void **pages, size_t npages, int prot) +{ + struct mm_map_entry_s entry; + FAR void *vaddr; + size_t size; + int ret; + + /* The region size is full pages */ + + size = npages << MM_PGSHIFT; + + /* Find a virtual memory area that fits */ + + vaddr = gran_alloc(&g_kmm_map_vpages, size); + if (!vaddr) + { + return NULL; + } + + /* Map the pages into the kernel page directory */ + + ret = up_addrenv_kmap_pages(pages, npages, (uintptr_t)vaddr, prot); + if (ret < 0) + { + goto errout_with_vaddr; + } + + entry.vaddr = vaddr; + entry.length = size; + entry.offset = 0; + entry.munmap = NULL; + + ret = mm_map_add(&g_kmm_map, &entry); + if (ret < 0) + { + goto errout_with_pgmap; + } + + return vaddr; + +errout_with_pgmap: + up_addrenv_kunmap_pages((uintptr_t)vaddr, npages); +errout_with_vaddr: + gran_free(&g_kmm_map_vpages, vaddr, size); + return NULL; +} + +/**************************************************************************** + * Name: map_single_user_page + * + * Description: + * Map a single user page into kernel memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +static void *map_single_user_page(uintptr_t vaddr) +{ + FAR struct tcb_s *tcb = nxsched_self(); + uintptr_t page; + + /* Find the page associated with this virtual address */ + + page = up_addrenv_find_page(&tcb->addrenv_own->addrenv, vaddr); + if (!page) + { + return NULL; + } + + vaddr = up_addrenv_page_vaddr(page); + return (FAR void *)vaddr; +} + +/**************************************************************************** + * Name: kmm_map_lock + * + * Description: + * Get exclusive access to the kernel mm_map + * + ****************************************************************************/ + +static int kmm_map_lock(void) +{ + return nxrmutex_lock(&g_kmm_map.mm_map_mutex); +} + +/**************************************************************************** + * Name: kmm_map_unlock + * + * Description: + * Relinquish exclusive access to the kernel mm_map + * + ****************************************************************************/ + +static void kmm_map_unlock(void) +{ + DEBUGVERIFY(nxrmutex_unlock(&g_kmm_map.mm_map_mutex)); +} + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: kmm_map_initialize + * + * Description: + * Initialize the kernel dynamic mapping module. + * + * Input Parameters: + * None. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_map_initialize(void) +{ + g_kmm_map_vpages = gran_initialize((FAR void *)CONFIG_ARCH_KMAP_VBASE, + CONFIG_ARCH_KMAP_NPAGES << MM_PGSHIFT, + MM_PGSHIFT, MM_PGSHIFT); + DEBUGVERIFY(g_kmm_map_vpages); + mm_map_initialize(&g_kmm_map, true); +} + +/**************************************************************************** + * Name: kmm_map_pages + * + * Description: + * Map pages into kernel virtual memory. + * + * Input Parameters: + * pages - Pointer to buffer that contains the physical page addresses. + * npages - Amount of pages. + * prot - Access right flags. + * + * Returned Value: + * Pointer to the mapped virtual memory on success; NULL on failure + * + ****************************************************************************/ + +FAR void *kmm_map(FAR void **pages, size_t npages, int prot) +{ + uintptr_t vaddr; + + if (!pages || !npages || npages > CONFIG_ARCH_KMAP_NPAGES) + { + return NULL; + } + + /* Attempt to map the pages */ + + vaddr = (uintptr_t)map_pages(pages, npages, prot); + if (!vaddr) + { + return NULL; + } + + return (FAR void *)(vaddr); +} + +/**************************************************************************** + * Name: kmm_unmap + * + * Description: + * Unmap a previously allocated kernel virtual memory area. + * + * Input Parameters: + * kaddr - The kernel virtual address where the mapping begins. + * + * Returned Value: + * None. + * + ****************************************************************************/ + +void kmm_unmap(FAR void *kaddr) +{ + FAR struct mm_map_entry_s *entry; + unsigned int npages; + int ret; + + /* Lock the mapping list when we fiddle around with it */ + + ret = kmm_map_lock(); + if (ret == OK) + { + /* Find the entry, it is OK if none found */ + + entry = mm_map_find(get_current_mm(), kaddr, 1); + if (entry) + { + npages = MM_NPAGES(entry->length); + + /* Remove the mappings from the page directory */ + + up_addrenv_kunmap_pages((uintptr_t)entry->vaddr, npages); + + /* Release the virtual memory area for use */ + + gran_free(&g_kmm_map_vpages, entry->vaddr, entry->length); + + /* Remove the mapping from the kernel mapping list */ + + mm_map_remove(&g_kmm_map, entry); + } + + kmm_map_unlock(); + } +} + +/**************************************************************************** + * Name: kmm_user_map + * + * Description: + * Map a region of user memory (physical pages) for kernel use through + * a continuous virtual memory area. + * + * Input Parameters: + * uaddr - The user virtual address where mapping begins. + * size - Size of the region. + * + * Returned Value: + * Pointer to the virtual memory area, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_user_map(FAR void *uaddr, size_t size) +{ + FAR void *pages; + uintptr_t vaddr; + uintptr_t offset; + size_t npages; + int ret; + + /* Find the kernel addressable virtual address, or map the user memory */ + + vaddr = (uintptr_t)uaddr; + + /* If the memory is not user memory, get out */ + + if (!up_addrenv_user_vaddr(vaddr)) + { + /* Not user memory, get out */ + + return uaddr; + } + + /* How many pages (including partial) does the space encompass ? */ + + offset = vaddr & MM_PGMASK; + vaddr = MM_PGALIGNDOWN(vaddr); + npages = MM_NPAGES(offset + size); + + /* Does the area fit in 1 page, including page boundary crossings ? */ + + if (npages == 1) + { + /* Yes, can simply return the kernel addressable virtual address */ + + vaddr = (uintptr_t)map_single_user_page(vaddr); + return (FAR void *)(vaddr + offset); + } + + /* No, the area must be mapped into kernel virtual address space */ + + pages = kmm_zalloc(npages * sizeof(void *)); + if (!pages) + { + return NULL; + } + + /* Fetch the physical pages for the user virtual address range */ + + ret = get_user_pages(&pages, npages, vaddr); + if (ret < 0) + { + goto errout_with_pages; + } + + /* Map the physical pages to kernel memory */ + + vaddr = (uintptr_t)map_pages(&pages, npages, PROT_READ | PROT_WRITE); + if (!vaddr) + { + goto errout_with_pages; + } + + /* Ok, we have a virtual memory area, add the offset back */ + + return (FAR void *)(vaddr + offset); + +errout_with_pages: + kmm_free(pages); + return NULL; +} + +/**************************************************************************** + * Name: kmm_map_user_page + * + * Description: + * Map a single physical page into kernel virtual memory. Typically just + * returns the kernel addressable page pool virtual address. + * + * Input Parameters: + * uaddr - The virtual address of the user page. + * + * Returned Value: + * Pointer to the new address environment, or NULL if out of memory. + * + ****************************************************************************/ + +FAR void *kmm_map_user_page(FAR void *uaddr) +{ + uintptr_t vaddr; + uintptr_t offset; + + /* Find the kernel addressable virtual address, or map the user memory */ + + vaddr = (uintptr_t)uaddr; + + /* If the memory is not user memory, get out */ + + if (!up_addrenv_user_vaddr(vaddr)) + { + /* Not user memory, get out */ + + return uaddr; + } + + /* Record the offset and add it back later */ + + offset = vaddr & MM_PGMASK; + vaddr = MM_PGALIGNDOWN(vaddr); + + vaddr = (uintptr_t)map_single_user_page(vaddr); + if (!vaddr) + { + return NULL; + } + + return (FAR void *)(vaddr + offset); +} + +#endif diff --git a/sched/init/nx_start.c b/sched/init/nx_start.c index df74373a0a..9bb37f965c 100644 --- a/sched/init/nx_start.c +++ b/sched/init/nx_start.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -472,6 +473,12 @@ void nx_start(void) } #endif +#ifdef CONFIG_MM_KMAP + /* Initialize the kernel dynamic mapping module */ + + kmm_map_initialize(); +#endif + #ifdef CONFIG_ARCH_HAVE_EXTRA_HEAPS /* Initialize any extra heap. */