sergiotarxz/nuttx
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

libc/x86_64: port string functions from bionic

Browse Source 
port optimized string functions for x86_64 from Bionic (BSD licensed)

Signed-off-by: p-szafonimateusz <p-szafonimateusz@xiaomi.com>
This commit is contained in:
p-szafonimateusz 2024-05-22 16:42:23 +02:00 committed by archer
parent fc716802c0
commit e6553eee5a
22 changed files with 7473 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
LICENSE
View File

@ -8634,3 +8634,43 @@ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
SUCH DAMAGE.
libs/libc/machine/x86_64/gnu/arch_memcmp.S
libs/libc/machine/x86_64/gnu/arch_memset_avx2.S
libs/libc/machine/x86_64/gnu/arch_memset_sse2.S
libs/libc/machine/x86_64/gnu/arch_stpcpy.S
libs/libc/machine/x86_64/gnu/arch_stpncpy.S
libs/libc/machine/x86_64/gnu/arch_strcat.S
libs/libc/machine/x86_64/gnu/arch_strcpy.S
libs/libc/machine/x86_64/gnu/arch_strlen.S
libs/libc/machine/x86_64/gnu/arch_strncmp.S
libs/libc/machine/x86_64/gnu/arch_strncpy.S
libs/libc/machine/x86_64/gnu/cache.h
======================
Copyright (c) 2014, Intel Corporation
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* Neither the name of Intel Corporation nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

1
arch/Kconfig
View File

@ -138,6 +138,7 @@ config ARCH_X86_64
select ARCH_HAVE_INTERRUPTSTACK
select ARCH_HAVE_CUSTOMOPT
select LIBC_ARCH_ELF_64BIT if LIBC_ARCH_ELF
select ARCH_TOOLCHAIN_GNU
---help---
x86-64 architectures.

11
libs/libc/machine/Kconfig
View File

@ -128,6 +128,14 @@ config LIBC_ARCH_STRNCASECMP
bool
default n
config LIBC_ARCH_STPCPY
bool
default n
config LIBC_ARCH_STPNCPY
bool
default n
config LIBC_ARCH_ELF
bool
default n
@ -788,6 +796,9 @@ endif
if ARCH_X86
source "libs/libc/machine/x86/Kconfig"
endif
if ARCH_X86_64
source "libs/libc/machine/x86_64/Kconfig"
endif
if ARCH_XTENSA
source "libs/libc/machine/xtensa/Kconfig"
endif

3
libs/libc/machine/Make.defs
View File

@ -37,6 +37,9 @@ endif
ifeq ($(CONFIG_ARCH_X86),y)
include $(TOPDIR)/libs/libc/machine/x86/Make.defs
endif
ifeq ($(CONFIG_ARCH_X86_64),y)
include $(TOPDIR)/libs/libc/machine/x86_64/Make.defs
endif
ifeq ($(CONFIG_ARCH_XTENSA),y)
include $(TOPDIR)/libs/libc/machine/xtensa/Make.defs
endif

6
libs/libc/machine/x86_64/CMakeLists.txt
View File

@ -17,3 +17,9 @@
# the License.
#
# ##############################################################################
add_subdirectory(gnu)
set(SRCS)
target_sources(c PRIVATE ${SRCS})

91
libs/libc/machine/x86_64/Kconfig
View File

@ -2,3 +2,94 @@
# For a description of the syntax of this configuration file,
# see the file kconfig-language.txt in the NuttX tools repository.
#
if ARCH_TOOLCHAIN_GNU && ALLOW_BSD_COMPONENTS
config X86_64_MEMCMP
bool "Enable optimized memcmp() for X86_64"
select LIBC_ARCH_MEMCMP
depends on ARCH_HAVE_SSE41
---help---
Enable optimized X86_64 specific memcmp() library function
config X86_64_MEMMOVE
bool "Enable optimized memmove()/memcpy() for X86_64"
default n
select LIBC_ARCH_MEMMOVE
select LIBC_ARCH_MEMCPY
---help---
Enable optimized X86_64 specific memmove()/memcpy() library function
config X86_64_MEMSET
bool "Enable optimized memset() for X86_64"
default n
select LIBC_ARCH_MEMSET
---help---
Enable optimized X86_64 specific memset() library function
config X86_64_STPCPY
bool "Enable optimized stpcpy() for X86_64"
default n
select LIBC_ARCH_STPCPY
---help---
Enable optimized X86_64 specific stpcpy() library function
config X86_64_STPNCPY
bool "Enable optimized stpncpy() for X86_64"
default n
select LIBC_ARCH_STPNCPY
---help---
Enable optimized X86_64 specific stpncpy() library function
config X86_64_STRCAT
bool "Enable optimized strcat() for X86_64"
default n
select LIBC_ARCH_STRCAT
---help---
Enable optimized X86_64 specific strcat() library function
config X86_64_STRCMP
bool "Enable optimized strcmp() for X86_64"
default n
select LIBC_ARCH_STRCMP
depends on ARCH_HAVE_SSSE3
---help---
Enable optimized X86_64 specific strcmp() library function
config X86_64_STRNCMP
bool "Enable optimized strncmp() for X86_64"
default n
select LIBC_ARCH_STRNCMP
depends on ARCH_HAVE_SSSE3
---help---
Enable optimized X86_64 specific strncmp() library function
config X86_64_STRCPY
bool "Enable optimized strcpy() for X86_64"
default n
select LIBC_ARCH_STRCPY
---help---
Enable optimized X86_64 specific strcpy() library function
config X86_64_STRLEN
bool "Enable optimized strlen() for X86_64"
default n
select LIBC_ARCH_STRLEN
---help---
Enable optimized X86_64 specific strlen() library function
config X86_64_STRNCPY
bool "Enable optimized strncpy() for X86_64"
default n
select LIBC_ARCH_STRNCPY
---help---
Enable optimized X86_64 specific strncpy() library function
config X86_64_STRNCMP
bool "Enable optimized strncmp() for X86_64"
default n
select LIBC_ARCH_STRNCMP
---help---
Enable optimized X86_64 specific strncmp() library function
endif # ARCH_TOOLCHAIN_GNU && ALLOW_BSD_COMPONENTS

75
libs/libc/machine/x86_64/Make.defs Normal file
View File

@ -0,0 +1,75 @@
############################################################################
# libs/libc/machine/x86_64/Make.defs
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership. The
# ASF licenses this file to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
############################################################################
ifeq ($(CONFIG_X86_64_MEMCMP),y)
ASRCS += arch_memcmp.S
endif
ifeq ($(CONFIG_X86_64_MEMMOVE),y)
ASRCS += arch_memmove.S
endif
ifeq ($(CONFIG_X86_64_MEMSET),y)
ifeq ($(CONFIG_ARCH_X86_64_AVX),y)
ASRCS += arch_memset_avx2.S
else
ASRCS += arch_memset_sse2.S
endif
endif
ifeq ($(CONFIG_X86_64_STPCPY),y)
ASRCS += arch_stpcpy.S
endif
ifeq ($(CONFIG_X86_64_STPNCPY),y)
ASRCS += arch_stpncpy.S
endif
ifeq ($(CONFIG_X86_64_STRCAT),y)
ASRCS += arch_strcat.S
endif
ifeq ($(CONFIG_X86_64_STRCMP),y)
ASRCS += arch_strcmp.S
endif
ifeq ($(CONFIG_X86_64_STRCPY),y)
ASRCS += arch_strcpy.S
endif
ifeq ($(CONFIG_X86_64_STRLEN),y)
ASRCS += arch_strlen.S
endif
ifeq ($(CONFIG_X86_64_STRNCPY),y)
ASRCS += arch_strncpy.S
endif
ifeq ($(CONFIG_X86_64_STRNCMP),y)
ASRCS += arch_strncmp.S
endif
ifeq ($(CONFIG_ARCH_TOOLCHAIN_GNU),y)
DEPPATH += --dep-path machine/x86_64/gnu
VPATH += :machine/x86_64/gnu
endif
DEPPATH += --dep-path machine/x86_64
VPATH += :machine/x86_64

71
libs/libc/machine/x86_64/gnu/CMakeLists.txt Normal file
View File

@ -0,0 +1,71 @@
# ##############################################################################
# libs/libc/machine/X86_64/gnu/CMakeLists.txt
#
# Licensed to the Apache Software Foundation (ASF) under one or more contributor
# license agreements. See the NOTICE file distributed with this work for
# additional information regarding copyright ownership. The ASF licenses this
# file to you under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
#
# ##############################################################################
set(SRCS)
if(CONFIG_X86_64_MEMCMP)
list(APPEND SRCS arch_memcmp.S)
endif()
if(CONFIG_X86_64_MEMMOVE)
list(APPEND SRCS arch_memmove.S)
endif()
if(CONFIG_X86_64_MEMSET)
if(CONFIG_ARCH_X86_64_AVX)
list(APPEND SRCS arch_memset_avx2.S)
else()
list(APPEND SRCS arch_memset_sse2.S)
endif()
endif()
if(CONFIG_X86_64_STPCPY)
list(APPEND SRCS arch_stpcpy.S)
endif()
if(CONFIG_X86_64_STPNCPY)
list(APPEND SRCS arch_stpncpy.S)
endif()
if(CONFIG_X86_64_STRCAT)
list(APPEND SRCS arch_strcat.S)
endif()
if(CONFIG_X86_64_STRCMP)
list(APPEND SRCS arch_strcmp.S)
endif()
if(CONFIG_X86_64_STRCPY)
list(APPEND SRCS arch_strcpy.S)
endif()
if(CONFIG_X86_64_STRLEN)
list(APPEND SRCS arch_strlen.S)
endif()
if(CONFIG_X86_64_STRNCPY)
list(APPEND SRCS arch_strncpy.S)
endif()
if(CONFIG_X86_64_STRNCMP)
list(APPEND SRCS arch_strncmp.S)
endif()
target_sources(c PRIVATE ${SRCS})

1814
libs/libc/machine/x86_64/gnu/arch_memcmp.S Normal file
View File

File diff suppressed because it is too large Load Diff

533
libs/libc/machine/x86_64/gnu/arch_memmove.S Normal file
View File

@ -0,0 +1,533 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_memmove.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/*********************************************************************************
* Included Files
*********************************************************************************/
#include "cache.h"
/*********************************************************************************
* Pre-processor Definitions
*********************************************************************************/
#ifndef MEMMOVE
# define MEMMOVE memmove
#endif
#ifndef L
# define L(label) .L##label
#endif
#ifndef cfi_startproc
# define cfi_startproc .cfi_startproc
#endif
#ifndef cfi_endproc
# define cfi_endproc .cfi_endproc
#endif
#ifndef cfi_rel_offset
# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
#endif
#ifndef cfi_restore
# define cfi_restore(reg) .cfi_restore reg
#endif
#ifndef cfi_adjust_cfa_offset
# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
#endif
#ifndef ENTRY
# define ENTRY(name) \
.type name, @function; \
.globl name; \
.p2align 4; \
name: \
cfi_startproc
#endif
#ifndef ALIAS_SYMBOL
# define ALIAS_SYMBOL(alias, original) \
.globl alias; \
.equ alias, original
#endif
#ifndef END
# define END(name) \
cfi_endproc; \
.size name, .-name
#endif
#define CFI_PUSH(REG) \
cfi_adjust_cfa_offset (4); \
cfi_rel_offset (REG, 0)
#define CFI_POP(REG) \
cfi_adjust_cfa_offset (-4); \
cfi_restore (REG)
#define PUSH(REG) push REG;
#define POP(REG) pop REG;
#define ENTRANCE PUSH (%rbx);
#define RETURN_END POP (%rbx); ret
#define RETURN RETURN_END;
/*********************************************************************************
* Public Functions
*********************************************************************************/
.section .text.sse2,"ax",@progbits
ENTRY (MEMMOVE)
ENTRANCE
mov %rdi, %rax
/* Check whether we should copy backward or forward. */
cmp %rsi, %rdi
je L(mm_return)
jg L(mm_len_0_or_more_backward)
/* Now do checks for lengths. We do [0..16], [0..32], [0..64], [0..128]
separately. */
cmp $16, %rdx
jbe L(mm_len_0_16_bytes_forward)
cmp $32, %rdx
ja L(mm_len_32_or_more_forward)
/* Copy [0..32] and return. */
movdqu (%rsi), %xmm0
movdqu -16(%rsi, %rdx), %xmm1
movdqu %xmm0, (%rdi)
movdqu %xmm1, -16(%rdi, %rdx)
jmp L(mm_return)
L(mm_len_32_or_more_forward):
cmp $64, %rdx
ja L(mm_len_64_or_more_forward)
/* Copy [0..64] and return. */
movdqu (%rsi), %xmm0
movdqu 16(%rsi), %xmm1
movdqu -16(%rsi, %rdx), %xmm2
movdqu -32(%rsi, %rdx), %xmm3
movdqu %xmm0, (%rdi)
movdqu %xmm1, 16(%rdi)
movdqu %xmm2, -16(%rdi, %rdx)
movdqu %xmm3, -32(%rdi, %rdx)
jmp L(mm_return)
L(mm_len_64_or_more_forward):
cmp $128, %rdx
ja L(mm_len_128_or_more_forward)
/* Copy [0..128] and return. */
movdqu (%rsi), %xmm0
movdqu 16(%rsi), %xmm1
movdqu 32(%rsi), %xmm2
movdqu 48(%rsi), %xmm3
movdqu -64(%rsi, %rdx), %xmm4
movdqu -48(%rsi, %rdx), %xmm5
movdqu -32(%rsi, %rdx), %xmm6
movdqu -16(%rsi, %rdx), %xmm7
movdqu %xmm0, (%rdi)
movdqu %xmm1, 16(%rdi)
movdqu %xmm2, 32(%rdi)
movdqu %xmm3, 48(%rdi)
movdqu %xmm4, -64(%rdi, %rdx)
movdqu %xmm5, -48(%rdi, %rdx)
movdqu %xmm6, -32(%rdi, %rdx)
movdqu %xmm7, -16(%rdi, %rdx)
jmp L(mm_return)
L(mm_len_128_or_more_forward):
/* Aligning the address of destination. */
/* save first unaligned 64 bytes */
movdqu (%rsi), %xmm0
movdqu 16(%rsi), %xmm1
movdqu 32(%rsi), %xmm2
movdqu 48(%rsi), %xmm3
lea 64(%rdi), %r8
and $-64, %r8 /* r8 now aligned to next 64 byte boundary */
sub %rdi, %rsi /* rsi = src - dst = diff */
movdqu (%r8, %rsi), %xmm4
movdqu 16(%r8, %rsi), %xmm5
movdqu 32(%r8, %rsi), %xmm6
movdqu 48(%r8, %rsi), %xmm7
movdqu %xmm0, (%rdi)
movdqu %xmm1, 16(%rdi)
movdqu %xmm2, 32(%rdi)
movdqu %xmm3, 48(%rdi)
movdqa %xmm4, (%r8)
movaps %xmm5, 16(%r8)
movaps %xmm6, 32(%r8)
movaps %xmm7, 48(%r8)
add $64, %r8
lea (%rdi, %rdx), %rbx
and $-64, %rbx
cmp %r8, %rbx
jbe L(mm_copy_remaining_forward)
cmp $SHARED_CACHE_SIZE_HALF, %rdx
jae L(mm_large_page_loop_forward)
.p2align 4
L(mm_main_loop_forward):
prefetcht0 128(%r8, %rsi)
movdqu (%r8, %rsi), %xmm0
movdqu 16(%r8, %rsi), %xmm1
movdqu 32(%r8, %rsi), %xmm2
movdqu 48(%r8, %rsi), %xmm3
movdqa %xmm0, (%r8)
movaps %xmm1, 16(%r8)
movaps %xmm2, 32(%r8)
movaps %xmm3, 48(%r8)
lea 64(%r8), %r8
cmp %r8, %rbx
ja L(mm_main_loop_forward)
L(mm_copy_remaining_forward):
add %rdi, %rdx
sub %r8, %rdx
/* We copied all up till %rdi position in the dst.
In %rdx now is how many bytes are left to copy.
Now we need to advance %r8. */
lea (%r8, %rsi), %r9
L(mm_remaining_0_64_bytes_forward):
cmp $32, %rdx
ja L(mm_remaining_33_64_bytes_forward)
cmp $16, %rdx
ja L(mm_remaining_17_32_bytes_forward)
test %rdx, %rdx
.p2align 4,,2
je L(mm_return)
cmpb $8, %dl
ja L(mm_remaining_9_16_bytes_forward)
cmpb $4, %dl
.p2align 4,,5
ja L(mm_remaining_5_8_bytes_forward)
cmpb $2, %dl
.p2align 4,,1
ja L(mm_remaining_3_4_bytes_forward)
movzbl -1(%r9,%rdx), %esi
movzbl (%r9), %ebx
movb %sil, -1(%r8,%rdx)
movb %bl, (%r8)
jmp L(mm_return)
L(mm_remaining_33_64_bytes_forward):
movdqu (%r9), %xmm0
movdqu 16(%r9), %xmm1
movdqu -32(%r9, %rdx), %xmm2
movdqu -16(%r9, %rdx), %xmm3
movdqu %xmm0, (%r8)
movdqu %xmm1, 16(%r8)
movdqu %xmm2, -32(%r8, %rdx)
movdqu %xmm3, -16(%r8, %rdx)
jmp L(mm_return)
L(mm_remaining_17_32_bytes_forward):
movdqu (%r9), %xmm0
movdqu -16(%r9, %rdx), %xmm1
movdqu %xmm0, (%r8)
movdqu %xmm1, -16(%r8, %rdx)
jmp L(mm_return)
L(mm_remaining_5_8_bytes_forward):
movl (%r9), %esi
movl -4(%r9,%rdx), %ebx
movl %esi, (%r8)
movl %ebx, -4(%r8,%rdx)
jmp L(mm_return)
L(mm_remaining_9_16_bytes_forward):
mov (%r9), %rsi
mov -8(%r9, %rdx), %rbx
mov %rsi, (%r8)
mov %rbx, -8(%r8, %rdx)
jmp L(mm_return)
L(mm_remaining_3_4_bytes_forward):
movzwl -2(%r9,%rdx), %esi
movzwl (%r9), %ebx
movw %si, -2(%r8,%rdx)
movw %bx, (%r8)
jmp L(mm_return)
L(mm_len_0_16_bytes_forward):
testb $24, %dl
jne L(mm_len_9_16_bytes_forward)
testb $4, %dl
.p2align 4,,5
jne L(mm_len_5_8_bytes_forward)
test %rdx, %rdx
.p2align 4,,2
je L(mm_return)
testb $2, %dl
.p2align 4,,1
jne L(mm_len_2_4_bytes_forward)
movzbl -1(%rsi,%rdx), %ebx
movzbl (%rsi), %esi
movb %bl, -1(%rdi,%rdx)
movb %sil, (%rdi)
jmp L(mm_return)
L(mm_len_2_4_bytes_forward):
movzwl -2(%rsi,%rdx), %ebx
movzwl (%rsi), %esi
movw %bx, -2(%rdi,%rdx)
movw %si, (%rdi)
jmp L(mm_return)
L(mm_len_5_8_bytes_forward):
movl (%rsi), %ebx
movl -4(%rsi,%rdx), %esi
movl %ebx, (%rdi)
movl %esi, -4(%rdi,%rdx)
jmp L(mm_return)
L(mm_len_9_16_bytes_forward):
mov (%rsi), %rbx
mov -8(%rsi, %rdx), %rsi
mov %rbx, (%rdi)
mov %rsi, -8(%rdi, %rdx)
jmp L(mm_return)
L(mm_recalc_len):
/* Compute in %rdx how many bytes are left to copy after
the main loop stops. */
mov %rbx, %rdx
sub %rdi, %rdx
/* The code for copying backwards. */
L(mm_len_0_or_more_backward):
/* Now do checks for lengths. We do [0..16], [16..32], [32..64], [64..128]
separately. */
cmp $16, %rdx
jbe L(mm_len_0_16_bytes_backward)
cmp $32, %rdx
ja L(mm_len_32_or_more_backward)
/* Copy [0..32] and return. */
movdqu (%rsi), %xmm0
movdqu -16(%rsi, %rdx), %xmm1
movdqu %xmm0, (%rdi)
movdqu %xmm1, -16(%rdi, %rdx)
jmp L(mm_return)
L(mm_len_32_or_more_backward):
cmp $64, %rdx
ja L(mm_len_64_or_more_backward)
/* Copy [0..64] and return. */
movdqu (%rsi), %xmm0
movdqu 16(%rsi), %xmm1
movdqu -16(%rsi, %rdx), %xmm2
movdqu -32(%rsi, %rdx), %xmm3
movdqu %xmm0, (%rdi)
movdqu %xmm1, 16(%rdi)
movdqu %xmm2, -16(%rdi, %rdx)
movdqu %xmm3, -32(%rdi, %rdx)
jmp L(mm_return)
L(mm_len_64_or_more_backward):
cmp $128, %rdx
ja L(mm_len_128_or_more_backward)
/* Copy [0..128] and return. */
movdqu (%rsi), %xmm0
movdqu 16(%rsi), %xmm1
movdqu 32(%rsi), %xmm2
movdqu 48(%rsi), %xmm3
movdqu -64(%rsi, %rdx), %xmm4
movdqu -48(%rsi, %rdx), %xmm5
movdqu -32(%rsi, %rdx), %xmm6
movdqu -16(%rsi, %rdx), %xmm7
movdqu %xmm0, (%rdi)
movdqu %xmm1, 16(%rdi)
movdqu %xmm2, 32(%rdi)
movdqu %xmm3, 48(%rdi)
movdqu %xmm4, -64(%rdi, %rdx)
movdqu %xmm5, -48(%rdi, %rdx)
movdqu %xmm6, -32(%rdi, %rdx)
movdqu %xmm7, -16(%rdi, %rdx)
jmp L(mm_return)
L(mm_len_128_or_more_backward):
/* Aligning the address of destination. We need to save
16 bits from the source in order not to overwrite them. */
movdqu -16(%rsi, %rdx), %xmm0
movdqu -32(%rsi, %rdx), %xmm1
movdqu -48(%rsi, %rdx), %xmm2
movdqu -64(%rsi, %rdx), %xmm3
lea (%rdi, %rdx), %r9
and $-64, %r9 /* r9 = aligned dst */
mov %rsi, %r8
sub %rdi, %r8 /* r8 = src - dst, diff */
movdqu -16(%r9, %r8), %xmm4
movdqu -32(%r9, %r8), %xmm5
movdqu -48(%r9, %r8), %xmm6
movdqu -64(%r9, %r8), %xmm7
movdqu %xmm0, -16(%rdi, %rdx)
movdqu %xmm1, -32(%rdi, %rdx)
movdqu %xmm2, -48(%rdi, %rdx)
movdqu %xmm3, -64(%rdi, %rdx)
movdqa %xmm4, -16(%r9)
movaps %xmm5, -32(%r9)
movaps %xmm6, -48(%r9)
movaps %xmm7, -64(%r9)
lea -64(%r9), %r9
lea 64(%rdi), %rbx
and $-64, %rbx
cmp %r9, %rbx
jae L(mm_recalc_len)
cmp $SHARED_CACHE_SIZE_HALF, %rdx
jae L(mm_large_page_loop_backward)
.p2align 4
L(mm_main_loop_backward):
prefetcht0 -128(%r9, %r8)
movdqu -64(%r9, %r8), %xmm0
movdqu -48(%r9, %r8), %xmm1
movdqu -32(%r9, %r8), %xmm2
movdqu -16(%r9, %r8), %xmm3
movdqa %xmm0, -64(%r9)
movaps %xmm1, -48(%r9)
movaps %xmm2, -32(%r9)
movaps %xmm3, -16(%r9)
lea -64(%r9), %r9
cmp %r9, %rbx
jb L(mm_main_loop_backward)
jmp L(mm_recalc_len)
/* Copy [0..16] and return. */
L(mm_len_0_16_bytes_backward):
testb $24, %dl
jnz L(mm_len_9_16_bytes_backward)
testb $4, %dl
.p2align 4,,5
jnz L(mm_len_5_8_bytes_backward)
test %rdx, %rdx
.p2align 4,,2
je L(mm_return)
testb $2, %dl
.p2align 4,,1
jne L(mm_len_3_4_bytes_backward)
movzbl -1(%rsi,%rdx), %ebx
movzbl (%rsi), %ecx
movb %bl, -1(%rdi,%rdx)
movb %cl, (%rdi)
jmp L(mm_return)
L(mm_len_3_4_bytes_backward):
movzwl -2(%rsi,%rdx), %ebx
movzwl (%rsi), %ecx
movw %bx, -2(%rdi,%rdx)
movw %cx, (%rdi)
jmp L(mm_return)
L(mm_len_9_16_bytes_backward):
movl -4(%rsi,%rdx), %ebx
movl -8(%rsi,%rdx), %ecx
movl %ebx, -4(%rdi,%rdx)
movl %ecx, -8(%rdi,%rdx)
sub $8, %rdx
jmp L(mm_len_0_16_bytes_backward)
L(mm_len_5_8_bytes_backward):
movl (%rsi), %ebx
movl -4(%rsi,%rdx), %ecx
movl %ebx, (%rdi)
movl %ecx, -4(%rdi,%rdx)
L(mm_return):
RETURN
/* Big length copy forward part. */
.p2align 4
L(mm_large_page_loop_forward):
movdqu (%r8, %rsi), %xmm0
movdqu 16(%r8, %rsi), %xmm1
movdqu 32(%r8, %rsi), %xmm2
movdqu 48(%r8, %rsi), %xmm3
movntdq %xmm0, (%r8)
movntdq %xmm1, 16(%r8)
movntdq %xmm2, 32(%r8)
movntdq %xmm3, 48(%r8)
lea 64(%r8), %r8
cmp %r8, %rbx
ja L(mm_large_page_loop_forward)
sfence
jmp L(mm_copy_remaining_forward)
/* Big length copy backward part. */
.p2align 4
L(mm_large_page_loop_backward):
movdqu -64(%r9, %r8), %xmm0
movdqu -48(%r9, %r8), %xmm1
movdqu -32(%r9, %r8), %xmm2
movdqu -16(%r9, %r8), %xmm3
movntdq %xmm0, -64(%r9)
movntdq %xmm1, -48(%r9)
movntdq %xmm2, -32(%r9)
movntdq %xmm3, -16(%r9)
lea -64(%r9), %r9
cmp %r9, %rbx
jb L(mm_large_page_loop_backward)
sfence
jmp L(mm_recalc_len)
END (MEMMOVE)
ALIAS_SYMBOL(memcpy, MEMMOVE)

179
libs/libc/machine/x86_64/gnu/arch_memset_avx2.S Normal file
View File

@ -0,0 +1,179 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_memset_avx2.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/********************************************************************************
* Included Files
*********************************************************************************/
#include "cache.h"
/*********************************************************************************
* Pre-processor Definitions
*********************************************************************************/
#ifndef L
# define L(label) .L##label
#endif
#ifndef ALIGN
# define ALIGN(n) .p2align n
#endif
#define ENTRY(__f) \
.text; \
.global __f; \
.balign 16; \
.type __f, @function; \
__f: \
.cfi_startproc;
#define END(__f) \
.cfi_endproc; \
.size __f, .- __f;
/*********************************************************************************
* Public Functions
*********************************************************************************/
.section .text.avx2,"ax",@progbits
ENTRY(memset)
movq %rdi, %rax
and $0xff, %rsi
mov $0x0101010101010101, %rcx
imul %rsi, %rcx
cmpq $16, %rdx
jae L(16bytesormore)
testb $8, %dl
jnz L(8_15bytes)
testb $4, %dl
jnz L(4_7bytes)
testb $2, %dl
jnz L(2_3bytes)
testb $1, %dl
jz 1f
movb %cl, (%rdi)
1: ret
L(8_15bytes):
movq %rcx, (%rdi)
movq %rcx, -8(%rdi, %rdx)
ret
L(4_7bytes):
movl %ecx, (%rdi)
movl %ecx, -4(%rdi, %rdx)
ret
L(2_3bytes):
movw %cx, (%rdi)
movw %cx, -2(%rdi, %rdx)
ret
ALIGN (4)
L(16bytesormore):
movd %rcx, %xmm0
pshufd $0, %xmm0, %xmm0
movdqu %xmm0, (%rdi)
movdqu %xmm0, -16(%rdi, %rdx)
cmpq $32, %rdx
jbe L(done)
movdqu %xmm0, 16(%rdi)
movdqu %xmm0, -32(%rdi, %rdx)
cmpq $64, %rdx
jbe L(done)
movdqu %xmm0, 32(%rdi)
movdqu %xmm0, 48(%rdi)
movdqu %xmm0, -64(%rdi, %rdx)
movdqu %xmm0, -48(%rdi, %rdx)
cmpq $128, %rdx
jbe L(done)
vpbroadcastb %xmm0, %ymm0
vmovdqu %ymm0, 64(%rdi)
vmovdqu %ymm0, 96(%rdi)
vmovdqu %ymm0, -128(%rdi, %rdx)
vmovdqu %ymm0, -96(%rdi, %rdx)
cmpq $256, %rdx
jbe L(done)
ALIGN (4)
leaq 128(%rdi), %rcx
andq $-128, %rcx
movq %rdx, %r8
addq %rdi, %rdx
andq $-128, %rdx
cmpq %rcx, %rdx
je L(done)
#ifdef SHARED_CACHE_SIZE
cmp $SHARED_CACHE_SIZE, %r8
#else
cmp __x86_64_shared_cache_size(%rip), %r8
#endif
ja L(non_temporal_loop)
ALIGN (4)
L(normal_loop):
vmovdqa %ymm0, (%rcx)
vmovdqa %ymm0, 32(%rcx)
vmovdqa %ymm0, 64(%rcx)
vmovdqa %ymm0, 96(%rcx)
addq $128, %rcx
cmpq %rcx, %rdx
jne L(normal_loop)
jmp L(done)
ALIGN (4)
L(non_temporal_loop):
movntdq %xmm0, (%rcx)
movntdq %xmm0, 16(%rcx)
movntdq %xmm0, 32(%rcx)
movntdq %xmm0, 48(%rcx)
movntdq %xmm0, 64(%rcx)
movntdq %xmm0, 80(%rcx)
movntdq %xmm0, 96(%rcx)
movntdq %xmm0, 112(%rcx)
leaq 128(%rcx), %rcx
cmpq %rcx, %rdx
jne L(non_temporal_loop)
/* We used non-temporal stores, so we need a fence here. */
sfence
L(done):
/* We used the ymm registers, and that can break SSE2 performance
* unless you do this.
*/
vzeroupper
ret
END(memset)

166
libs/libc/machine/x86_64/gnu/arch_memset_sse2.S Normal file
View File

@ -0,0 +1,166 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_memset_sse2.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/********************************************************************************
* Included Files
*********************************************************************************/
#include "cache.h"
/*********************************************************************************
* Pre-processor Definitions
*********************************************************************************/
#ifndef L
# define L(label) .L##label
#endif
#ifndef ALIGN
# define ALIGN(n) .p2align n
#endif
#define ENTRY(__f) \
.text; \
.global __f; \
.balign 16; \
.type __f, @function; \
__f: \
.cfi_startproc;
#define END(__f) \
.cfi_endproc; \
.size __f, .- __f;
/*********************************************************************************
* Public Functions
*********************************************************************************/
.section .text.sse2,"ax",@progbits
ENTRY(memset)
movq %rdi, %rax
and $0xff, %rsi
mov $0x0101010101010101, %rcx
imul %rsi, %rcx
cmpq $16, %rdx
jae L(16bytesormore)
testb $8, %dl
jnz L(8_15bytes)
testb $4, %dl
jnz L(4_7bytes)
testb $2, %dl
jnz L(2_3bytes)
testb $1, %dl
jz L(return)
movb %cl, (%rdi)
L(return):
ret
L(8_15bytes):
movq %rcx, (%rdi)
movq %rcx, -8(%rdi, %rdx)
ret
L(4_7bytes):
movl %ecx, (%rdi)
movl %ecx, -4(%rdi, %rdx)
ret
L(2_3bytes):
movw %cx, (%rdi)
movw %cx, -2(%rdi, %rdx)
ret
ALIGN (4)
L(16bytesormore):
movd %rcx, %xmm0
pshufd $0, %xmm0, %xmm0
movdqu %xmm0, (%rdi)
movdqu %xmm0, -16(%rdi, %rdx)
cmpq $32, %rdx
jbe L(32bytesless)
movdqu %xmm0, 16(%rdi)
movdqu %xmm0, -32(%rdi, %rdx)
cmpq $64, %rdx
jbe L(64bytesless)
movdqu %xmm0, 32(%rdi)
movdqu %xmm0, 48(%rdi)
movdqu %xmm0, -64(%rdi, %rdx)
movdqu %xmm0, -48(%rdi, %rdx)
cmpq $128, %rdx
ja L(128bytesmore)
L(32bytesless):
L(64bytesless):
ret
ALIGN (4)
L(128bytesmore):
leaq 64(%rdi), %rcx
andq $-64, %rcx
movq %rdx, %r8
addq %rdi, %rdx
andq $-64, %rdx
cmpq %rcx, %rdx
je L(return)
#ifdef SHARED_CACHE_SIZE
cmp $SHARED_CACHE_SIZE, %r8
#else
cmp __x86_64_shared_cache_size(%rip), %r8
#endif
ja L(128bytesmore_nt)
ALIGN (4)
L(128bytesmore_normal):
movdqa %xmm0, (%rcx)
movaps %xmm0, 0x10(%rcx)
movaps %xmm0, 0x20(%rcx)
movaps %xmm0, 0x30(%rcx)
addq $64, %rcx
cmpq %rcx, %rdx
jne L(128bytesmore_normal)
ret
ALIGN (4)
L(128bytesmore_nt):
movntdq %xmm0, (%rcx)
movntdq %xmm0, 0x10(%rcx)
movntdq %xmm0, 0x20(%rcx)
movntdq %xmm0, 0x30(%rcx)
leaq 64(%rcx), %rcx
cmpq %rcx, %rdx
jne L(128bytesmore_nt)
sfence
ret
END(memset)

40
libs/libc/machine/x86_64/gnu/arch_stpcpy.S Normal file
View File

@ -0,0 +1,40 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_stpcpy.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/********************************************************************************
* Included Files
*********************************************************************************/
#define USE_AS_STPCPY
#define STRCPY stpcpy
#include "arch_strcpy.S"

41
libs/libc/machine/x86_64/gnu/arch_stpncpy.S Normal file
View File

@ -0,0 +1,41 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_strpcpy.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/********************************************************************************
* Included Files
*********************************************************************************/
#define USE_AS_STRNCPY
#define USE_AS_STPCPY
#define STRCPY stpncpy
#include "arch_strcpy.S"

98
libs/libc/machine/x86_64/gnu/arch_strcat.S Normal file
View File

@ -0,0 +1,98 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_strcat.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/*********************************************************************************
* Pre-processor Definitions
*********************************************************************************/
#ifndef STRCAT
# define STRCAT strcat
#endif
#ifndef L
# define L(label) .L##label
#endif
#ifndef cfi_startproc
# define cfi_startproc .cfi_startproc
#endif
#ifndef cfi_endproc
# define cfi_endproc .cfi_endproc
#endif
#ifndef ENTRY
# define ENTRY(name) \
.type name, @function; \
.globl name; \
.p2align 4; \
name: \
cfi_startproc
#endif
#ifndef END
# define END(name) \
cfi_endproc; \
.size name, .-name
#endif
#define USE_AS_STRCAT
/*********************************************************************************
* Public Functions
*********************************************************************************/
.text
ENTRY (STRCAT)
mov %rdi, %r9
#ifdef USE_AS_STRNCAT
mov %rdx, %r8
#endif
#define RETURN jmp L(Strcpy)
#include "arch_strlen.S"
#undef RETURN
#define RETURN ret
L(Strcpy):
lea (%r9, %rax), %rdi
mov %rsi, %rcx
mov %r9, %rax /* save result */
#ifdef USE_AS_STRNCAT
test %r8, %r8
jz L(ExitZero)
# define USE_AS_STRNCPY
#endif
#include "arch_strcpy.S"

1937
libs/libc/machine/x86_64/gnu/arch_strcmp.S Normal file
View File

File diff suppressed because it is too large Load Diff

1932
libs/libc/machine/x86_64/gnu/arch_strcpy.S Normal file
View File

File diff suppressed because it is too large Load Diff

305
libs/libc/machine/x86_64/gnu/arch_strlen.S Normal file
View File

@ -0,0 +1,305 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_strlen.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/*********************************************************************************
* Pre-processor Definitions
*********************************************************************************/
#ifndef USE_AS_STRCAT
#ifndef STRLEN
# define STRLEN strlen
#endif
#ifndef L
# define L(label) .L##label
#endif
#ifndef cfi_startproc
# define cfi_startproc .cfi_startproc
#endif
#ifndef cfi_endproc
# define cfi_endproc .cfi_endproc
#endif
#ifndef ENTRY
# define ENTRY(name) \
.type name, @function; \
.globl name; \
.p2align 4; \
name: \
cfi_startproc
#endif
#ifndef END
# define END(name) \
cfi_endproc; \
.size name, .-name
#endif
#define RETURN ret
/*********************************************************************************
* Public Functions
*********************************************************************************/
.section .text.sse2,"ax",@progbits
ENTRY (STRLEN)
/* end ifndef USE_AS_STRCAT */
#endif
xor %rax, %rax
mov %edi, %ecx
and $0x3f, %ecx
pxor %xmm0, %xmm0
cmp $0x30, %ecx
ja L(next)
movdqu (%rdi), %xmm1
pcmpeqb %xmm1, %xmm0
pmovmskb %xmm0, %edx
test %edx, %edx
jnz L(exit_less16)
mov %rdi, %rax
and $-16, %rax
jmp L(align16_start)
L(next):
mov %rdi, %rax
and $-16, %rax
pcmpeqb (%rax), %xmm0
mov $-1, %r10d
sub %rax, %rcx
shl %cl, %r10d
pmovmskb %xmm0, %edx
and %r10d, %edx
jnz L(exit)
L(align16_start):
pxor %xmm0, %xmm0
pxor %xmm1, %xmm1
pxor %xmm2, %xmm2
pxor %xmm3, %xmm3
pcmpeqb 16(%rax), %xmm0
pmovmskb %xmm0, %edx
test %edx, %edx
jnz L(exit16)
pcmpeqb 32(%rax), %xmm1
pmovmskb %xmm1, %edx
test %edx, %edx
jnz L(exit32)
pcmpeqb 48(%rax), %xmm2
pmovmskb %xmm2, %edx
test %edx, %edx
jnz L(exit48)
pcmpeqb 64(%rax), %xmm3
pmovmskb %xmm3, %edx
test %edx, %edx
jnz L(exit64)
pcmpeqb 80(%rax), %xmm0
add $64, %rax
pmovmskb %xmm0, %edx
test %edx, %edx
jnz L(exit16)
pcmpeqb 32(%rax), %xmm1
pmovmskb %xmm1, %edx
test %edx, %edx
jnz L(exit32)
pcmpeqb 48(%rax), %xmm2
pmovmskb %xmm2, %edx
test %edx, %edx
jnz L(exit48)
pcmpeqb 64(%rax), %xmm3
pmovmskb %xmm3, %edx
test %edx, %edx
jnz L(exit64)
pcmpeqb 80(%rax), %xmm0
add $64, %rax
pmovmskb %xmm0, %edx
test %edx, %edx
jnz L(exit16)
pcmpeqb 32(%rax), %xmm1
pmovmskb %xmm1, %edx
test %edx, %edx
jnz L(exit32)
pcmpeqb 48(%rax), %xmm2
pmovmskb %xmm2, %edx
test %edx, %edx
jnz L(exit48)
pcmpeqb 64(%rax), %xmm3
pmovmskb %xmm3, %edx
test %edx, %edx
jnz L(exit64)
pcmpeqb 80(%rax), %xmm0
add $64, %rax
pmovmskb %xmm0, %edx
test %edx, %edx
jnz L(exit16)
pcmpeqb 32(%rax), %xmm1
pmovmskb %xmm1, %edx
test %edx, %edx
jnz L(exit32)
pcmpeqb 48(%rax), %xmm2
pmovmskb %xmm2, %edx
test %edx, %edx
jnz L(exit48)
pcmpeqb 64(%rax), %xmm3
pmovmskb %xmm3, %edx
test %edx, %edx
jnz L(exit64)
test $0x3f, %rax
jz L(align64_loop)
pcmpeqb 80(%rax), %xmm0
add $80, %rax
pmovmskb %xmm0, %edx
test %edx, %edx
jnz L(exit)
test $0x3f, %rax
jz L(align64_loop)
pcmpeqb 16(%rax), %xmm1
add $16, %rax
pmovmskb %xmm1, %edx
test %edx, %edx
jnz L(exit)
test $0x3f, %rax
jz L(align64_loop)
pcmpeqb 16(%rax), %xmm2
add $16, %rax
pmovmskb %xmm2, %edx
test %edx, %edx
jnz L(exit)
test $0x3f, %rax
jz L(align64_loop)
pcmpeqb 16(%rax), %xmm3
add $16, %rax
pmovmskb %xmm3, %edx
test %edx, %edx
jnz L(exit)
add $16, %rax
.p2align 4
L(align64_loop):
movaps (%rax), %xmm4
pminub 16(%rax), %xmm4
movaps 32(%rax), %xmm5
pminub 48(%rax), %xmm5
add $64, %rax
pminub %xmm4, %xmm5
pcmpeqb %xmm0, %xmm5
pmovmskb %xmm5, %edx
test %edx, %edx
jz L(align64_loop)
pcmpeqb -64(%rax), %xmm0
sub $80, %rax
pmovmskb %xmm0, %edx
test %edx, %edx
jnz L(exit16)
pcmpeqb 32(%rax), %xmm1
pmovmskb %xmm1, %edx
test %edx, %edx
jnz L(exit32)
pcmpeqb 48(%rax), %xmm2
pmovmskb %xmm2, %edx
test %edx, %edx
jnz L(exit48)
pcmpeqb 64(%rax), %xmm3
pmovmskb %xmm3, %edx
sub %rdi, %rax
bsf %rdx, %rdx
add %rdx, %rax
add $64, %rax
RETURN
.p2align 4
L(exit):
sub %rdi, %rax
L(exit_less16):
bsf %rdx, %rdx
add %rdx, %rax
RETURN
.p2align 4
L(exit16):
sub %rdi, %rax
bsf %rdx, %rdx
add %rdx, %rax
add $16, %rax
RETURN
.p2align 4
L(exit32):
sub %rdi, %rax
bsf %rdx, %rdx
add %rdx, %rax
add $32, %rax
RETURN
.p2align 4
L(exit48):
sub %rdi, %rax
bsf %rdx, %rdx
add %rdx, %rax
add $48, %rax
RETURN
.p2align 4
L(exit64):
sub %rdi, %rax
bsf %rdx, %rdx
add %rdx, %rax
add $64, %rax
#ifndef USE_AS_STRCAT
RETURN
END (STRLEN)
#endif

40
libs/libc/machine/x86_64/gnu/arch_strncmp.S Normal file
View File

@ -0,0 +1,40 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_strncmp.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/********************************************************************************
* Included Files
*********************************************************************************/
#define USE_AS_STRNCMP
#define STRCMP strncmp
#include "arch_strcmp.S"

40
libs/libc/machine/x86_64/gnu/arch_strncpy.S Normal file
View File

@ -0,0 +1,40 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/arch_strncpy.S
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
/********************************************************************************
* Included Files
*********************************************************************************/
#define USE_AS_STRNCPY
#define STRCPY strncpy
#include "arch_strcpy.S"

49
libs/libc/machine/x86_64/gnu/cache.h Normal file
View File

@ -0,0 +1,49 @@
/*********************************************************************************
* libs/libc/machine/x86_64/gnu/cache.h
*
* Copyright (c) 2014, Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* * this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* * this list of conditions and the following disclaimer in the documentation
* * and/or other materials provided with the distribution.
*
* * Neither the name of Intel Corporation nor the names of its contributors
* * may be used to endorse or promote products derived from this software
* * without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*********************************************************************************/
#ifndef __LIBS_LIBC_MACHINE_X86_64_GNU_CACHE_H
#define __LIBS_LIBC_MACHINE_X86_64_GNU_CACHE_H
/*********************************************************************************
* Pre-processor Definitions
*********************************************************************************/
/* Values are optimized for Core Architecture */
#define SHARED_CACHE_SIZE (4096 * 1024) /* Core Architecture L2 Cache */
#define DATA_CACHE_SIZE (24 * 1024) /* Core Architecture L1 Data Cache */
#define SHARED_CACHE_SIZE_HALF (SHARED_CACHE_SIZE / 2)
#define DATA_CACHE_SIZE_HALF (DATA_CACHE_SIZE / 2)
#endif /* __LIBS_LIBC_MACHINE_X86_64_GNU_CACHE_H */

2
libs/libc/string/lib_stpcpy.c
View File

@ -43,7 +43,7 @@
*
****************************************************************************/
#ifndef CONFIG_ARCH_STPCPY
#ifndef CONFIG_LIBC_ARCH_STPCPY
#undef stpcpy /* See mm/README.txt */
FAR char *stpcpy(FAR char *dest, FAR const char *src)
{