⚝
One Hat Cyber Team
⚝
Your IP:
216.73.216.96
Server IP:
147.93.97.220
Server:
Linux srv843233 6.8.0-71-generic #71-Ubuntu SMP PREEMPT_DYNAMIC Tue Jul 22 16:52:38 UTC 2025 x86_64
Server Software:
nginx/1.28.0
PHP Version:
8.2.29
Buat File
|
Buat Folder
Eksekusi
Dir :
~
/
lib
/
x86_64-linux-gnu
/
perl
/
5.38.2
/
CORE
/
Edit File: inline.h
/* inline.h * * Copyright (C) 2012 by Larry Wall and others * * You may distribute under the terms of either the GNU General Public * License or the Artistic License, as specified in the README file. * * This file contains tables and code adapted from * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which requires this * copyright notice: Copyright (c) 2008-2009 Bjoern Hoehrmann
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * * This file is a home for static inline functions that cannot go in other * header files, because they depend on proto.h (included after most other * headers) or struct definitions. * * Note also perlstatic.h for functions that can't or shouldn't be inlined, but * whose details should be exposed to the compiler, for such things as tail * call optimization. * * Each section names the header file that the functions "belong" to. */ /* ------------------------------- av.h ------------------------------- */ /* =for apidoc_section $AV =for apidoc av_count Returns the number of elements in the array C
. This is the true length of the array, including any undefined elements. It is always the same as S
>. =cut */ PERL_STATIC_INLINE Size_t Perl_av_count(pTHX_ AV *av) { PERL_ARGS_ASSERT_AV_COUNT; assert(SvTYPE(av) == SVt_PVAV); return AvFILL(av) + 1; } /* ------------------------------- av.c ------------------------------- */ /* =for apidoc av_store_simple This is a cut-down version of av_store that assumes that the array is very straightforward - no magic, not readonly, and AvREAL - and that C
is not negative. This function MUST NOT be used in situations where any of those assumptions may not hold. Stores an SV in an array. The array index is specified as C
. It can be dereferenced to get the C
that was stored there (= C
)). Note that the caller is responsible for suitably incrementing the reference count of C
before the call. Approximate Perl equivalent: C
. =cut */ PERL_STATIC_INLINE SV** Perl_av_store_simple(pTHX_ AV *av, SSize_t key, SV *val) { SV** ary; PERL_ARGS_ASSERT_AV_STORE_SIMPLE; assert(SvTYPE(av) == SVt_PVAV); assert(!SvMAGICAL(av)); assert(!SvREADONLY(av)); assert(AvREAL(av)); assert(key > -1); ary = AvARRAY(av); if (AvFILLp(av) < key) { if (key > AvMAX(av)) { av_extend(av,key); ary = AvARRAY(av); } AvFILLp(av) = key; } else SvREFCNT_dec(ary[key]); ary[key] = val; return &ary[key]; } /* =for apidoc av_fetch_simple This is a cut-down version of av_fetch that assumes that the array is very straightforward - no magic, not readonly, and AvREAL - and that C
is not negative. This function MUST NOT be used in situations where any of those assumptions may not hold. Returns the SV at the specified index in the array. The C
is the index. If lval is true, you are guaranteed to get a real SV back (in case it wasn't real before), which you can then modify. Check that the return value is non-null before dereferencing it to a C
. The rough perl equivalent is C<$myarray[$key]>. =cut */ PERL_STATIC_INLINE SV** Perl_av_fetch_simple(pTHX_ AV *av, SSize_t key, I32 lval) { PERL_ARGS_ASSERT_AV_FETCH_SIMPLE; assert(SvTYPE(av) == SVt_PVAV); assert(!SvMAGICAL(av)); assert(!SvREADONLY(av)); assert(AvREAL(av)); assert(key > -1); if ( (key > AvFILLp(av)) || !AvARRAY(av)[key]) { return lval ? av_store_simple(av,key,newSV_type(SVt_NULL)) : NULL; } else { return &AvARRAY(av)[key]; } } /* =for apidoc av_push_simple This is a cut-down version of av_push that assumes that the array is very straightforward - no magic, not readonly, and AvREAL - and that C
is not less than -1. This function MUST NOT be used in situations where any of those assumptions may not hold. Pushes an SV (transferring control of one reference count) onto the end of the array. The array will grow automatically to accommodate the addition. Perl equivalent: C
. =cut */ PERL_STATIC_INLINE void Perl_av_push_simple(pTHX_ AV *av, SV *val) { PERL_ARGS_ASSERT_AV_PUSH_SIMPLE; assert(SvTYPE(av) == SVt_PVAV); assert(!SvMAGICAL(av)); assert(!SvREADONLY(av)); assert(AvREAL(av)); assert(AvFILLp(av) > -2); (void)av_store_simple(av,AvFILLp(av)+1,val); } /* =for apidoc av_new_alloc This implements L
> and L
>, which are the public API for this functionality. Creates a new AV and allocates its SV* array. This is similar to, but more efficient than doing: AV *av = newAV(); av_extend(av, key); The size parameter is used to pre-allocate a SV* array large enough to hold at least elements C<0..(size-1)>. C
must be at least 1. The C
parameter controls whether or not the array is NULL initialized. =cut */ PERL_STATIC_INLINE AV * Perl_av_new_alloc(pTHX_ SSize_t size, bool zeroflag) { AV * const av = newAV(); SV** ary; PERL_ARGS_ASSERT_AV_NEW_ALLOC; assert(size > 0); Newx(ary, size, SV*); /* Newx performs the memwrap check */ AvALLOC(av) = ary; AvARRAY(av) = ary; AvMAX(av) = size - 1; if (zeroflag) Zero(ary, size, SV*); return av; } /* ------------------------------- cv.h ------------------------------- */ /* =for apidoc_section $CV =for apidoc CvGV Returns the GV associated with the CV C
, reifying it if necessary. =cut */ PERL_STATIC_INLINE GV * Perl_CvGV(pTHX_ CV *sv) { PERL_ARGS_ASSERT_CVGV; return CvNAMED(sv) ? Perl_cvgv_from_hek(aTHX_ sv) : ((XPVCV*)MUTABLE_PTR(SvANY(sv)))->xcv_gv_u.xcv_gv; } /* =for apidoc CvDEPTH Returns the recursion level of the CV C
. Hence >= 2 indicates we are in a recursive call. =cut */ PERL_STATIC_INLINE I32 * Perl_CvDEPTH(const CV * const sv) { PERL_ARGS_ASSERT_CVDEPTH; assert(SvTYPE(sv) == SVt_PVCV || SvTYPE(sv) == SVt_PVFM); return &((XPVCV*)SvANY(sv))->xcv_depth; } /* CvPROTO returns the prototype as stored, which is not necessarily what the interpreter should be using. Specifically, the interpreter assumes that spaces have been stripped, which has been the case if the prototype was added by toke.c, but is generally not the case if it was added elsewhere. Since we can't enforce the spacelessness at assignment time, this routine provides a temporary copy at parse time with spaces removed. I
is the start of the original buffer, I
is the length of the prototype and will be updated when this returns. */ #ifdef PERL_CORE PERL_STATIC_INLINE char * S_strip_spaces(pTHX_ const char * orig, STRLEN * const len) { SV * tmpsv; char * tmps; tmpsv = newSVpvn_flags(orig, *len, SVs_TEMP); tmps = SvPVX(tmpsv); while ((*len)--) { if (!isSPACE(*orig)) *tmps++ = *orig; orig++; } *tmps = '\0'; *len = tmps - SvPVX(tmpsv); return SvPVX(tmpsv); } #endif /* ------------------------------- iperlsys.h ------------------------------- */ #if ! defined(PERL_IMPLICIT_SYS) && defined(USE_ITHREADS) /* Otherwise this function is implemented as macros in iperlsys.h */ PERL_STATIC_INLINE bool S_PerlEnv_putenv(pTHX_ char * str) { PERL_ARGS_ASSERT_PERLENV_PUTENV; ENV_LOCK; bool retval = putenv(str); ENV_UNLOCK; return retval; } #endif /* ------------------------------- mg.h ------------------------------- */ #if defined(PERL_CORE) || defined(PERL_EXT) /* assumes get-magic and stringification have already occurred */ PERL_STATIC_INLINE STRLEN S_MgBYTEPOS(pTHX_ MAGIC *mg, SV *sv, const char *s, STRLEN len) { assert(mg->mg_type == PERL_MAGIC_regex_global); assert(mg->mg_len != -1); if (mg->mg_flags & MGf_BYTES || !DO_UTF8(sv)) return (STRLEN)mg->mg_len; else { const STRLEN pos = (STRLEN)mg->mg_len; /* Without this check, we may read past the end of the buffer: */ if (pos > sv_or_pv_len_utf8(sv, s, len)) return len+1; return sv_or_pv_pos_u2b(sv, s, pos, NULL); } } #endif /* ------------------------------- pad.h ------------------------------ */ #if defined(PERL_IN_PAD_C) || defined(PERL_IN_OP_C) PERL_STATIC_INLINE bool S_PadnameIN_SCOPE(const PADNAME * const pn, const U32 seq) { PERL_ARGS_ASSERT_PADNAMEIN_SCOPE; /* is seq within the range _LOW to _HIGH ? * This is complicated by the fact that PL_cop_seqmax * may have wrapped around at some point */ if (COP_SEQ_RANGE_LOW(pn) == PERL_PADSEQ_INTRO) return FALSE; /* not yet introduced */ if (COP_SEQ_RANGE_HIGH(pn) == PERL_PADSEQ_INTRO) { /* in compiling scope */ if ( (seq > COP_SEQ_RANGE_LOW(pn)) ? (seq - COP_SEQ_RANGE_LOW(pn) < (U32_MAX >> 1)) : (COP_SEQ_RANGE_LOW(pn) - seq > (U32_MAX >> 1)) ) return TRUE; } else if ( (COP_SEQ_RANGE_LOW(pn) > COP_SEQ_RANGE_HIGH(pn)) ? ( seq > COP_SEQ_RANGE_LOW(pn) || seq <= COP_SEQ_RANGE_HIGH(pn)) : ( seq > COP_SEQ_RANGE_LOW(pn) && seq <= COP_SEQ_RANGE_HIGH(pn)) ) return TRUE; return FALSE; } #endif /* ------------------------------- pp.h ------------------------------- */ PERL_STATIC_INLINE I32 Perl_TOPMARK(pTHX) { DEBUG_s(DEBUG_v(PerlIO_printf(Perl_debug_log, "MARK top %p %" IVdf "\n", PL_markstack_ptr, (IV)*PL_markstack_ptr))); return *PL_markstack_ptr; } PERL_STATIC_INLINE I32 Perl_POPMARK(pTHX) { DEBUG_s(DEBUG_v(PerlIO_printf(Perl_debug_log, "MARK pop %p %" IVdf "\n", (PL_markstack_ptr-1), (IV)*(PL_markstack_ptr-1)))); assert((PL_markstack_ptr > PL_markstack) || !"MARK underflow"); return *PL_markstack_ptr--; } /* ----------------------------- regexp.h ----------------------------- */ /* PVLVs need to act as a superset of all scalar types - they are basically * PVMGs with a few extra fields. * REGEXPs are first class scalars, but have many fields that can't be copied * into a PVLV body. * * Hence we take a different approach - instead of a copy, PVLVs store a pointer * back to the original body. To avoid increasing the size of PVLVs just for the * rare case of REGEXP assignment, this pointer is stored in the memory usually * used for SvLEN(). Hence the check for SVt_PVLV below, and the ? : ternary to * read the pointer from the two possible locations. The macro SvLEN() wraps the * access to the union's member xpvlenu_len, but there is no equivalent macro * for wrapping the union's member xpvlenu_rx, hence the direct reference here. * * See commit df6b4bd56551f2d3 for more details. */ PERL_STATIC_INLINE struct regexp * Perl_ReANY(const REGEXP * const re) { XPV* const p = (XPV*)SvANY(re); PERL_ARGS_ASSERT_REANY; assert(isREGEXP(re)); return SvTYPE(re) == SVt_PVLV ? p->xpv_len_u.xpvlenu_rx : (struct regexp *)p; } /* ------------------------------- utf8.h ------------------------------- */ /* =for apidoc_section $unicode */ PERL_STATIC_INLINE void Perl_append_utf8_from_native_byte(const U8 byte, U8** dest) { /* Takes an input 'byte' (Latin1 or EBCDIC) and appends it to the UTF-8 * encoded string at '*dest', updating '*dest' to include it */ PERL_ARGS_ASSERT_APPEND_UTF8_FROM_NATIVE_BYTE; if (NATIVE_BYTE_IS_INVARIANT(byte)) *((*dest)++) = byte; else { *((*dest)++) = UTF8_EIGHT_BIT_HI(byte); *((*dest)++) = UTF8_EIGHT_BIT_LO(byte); } } /* =for apidoc valid_utf8_to_uvchr Like C
>, but should only be called when it is known that the next character in the input UTF-8 string C
is well-formed (I
, it passes C
>. Surrogates, non-character code points, and non-Unicode code points are allowed. =cut */ PERL_STATIC_INLINE UV Perl_valid_utf8_to_uvchr(const U8 *s, STRLEN *retlen) { const UV expectlen = UTF8SKIP(s); const U8* send = s + expectlen; UV uv = *s; PERL_ARGS_ASSERT_VALID_UTF8_TO_UVCHR; if (retlen) { *retlen = expectlen; } /* An invariant is trivially returned */ if (expectlen == 1) { return uv; } /* Remove the leading bits that indicate the number of bytes, leaving just * the bits that are part of the value */ uv = NATIVE_UTF8_TO_I8(uv) & UTF_START_MASK(expectlen); /* Now, loop through the remaining bytes, accumulating each into the * working total as we go. (I khw tried unrolling the loop for up to 4 * bytes, but there was no performance improvement) */ for (++s; s < send; s++) { uv = UTF8_ACCUMULATE(uv, *s); } return UNI_TO_NATIVE(uv); } /* =for apidoc is_utf8_invariant_string Returns TRUE if the first C
bytes of the string C
are the same regardless of the UTF-8 encoding of the string (or UTF-EBCDIC encoding on EBCDIC machines); otherwise it returns FALSE. That is, it returns TRUE if they are UTF-8 invariant. On ASCII-ish machines, all the ASCII characters and only the ASCII characters fit this definition. On EBCDIC machines, the ASCII-range characters are invariant, but so also are the C1 controls. If C
is 0, it will be calculated using C
, (which means if you use this option, that C
can't have embedded C
characters and has to have a terminating C
byte). See also C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, and C
>. =cut */ #define is_utf8_invariant_string(s, len) \ is_utf8_invariant_string_loc(s, len, NULL) /* =for apidoc is_utf8_invariant_string_loc Like C
> but upon failure, stores the location of the first UTF-8 variant character in the C
pointer; if all characters are UTF-8 invariant, this function does not change the contents of C<*ep>. =cut */ PERL_STATIC_INLINE bool Perl_is_utf8_invariant_string_loc(const U8* const s, STRLEN len, const U8 ** ep) { const U8* send; const U8* x = s; PERL_ARGS_ASSERT_IS_UTF8_INVARIANT_STRING_LOC; if (len == 0) { len = strlen((const char *)s); } send = s + len; /* This looks like 0x010101... */ # define PERL_COUNT_MULTIPLIER (~ (UINTMAX_C(0)) / 0xFF) /* This looks like 0x808080... */ # define PERL_VARIANTS_WORD_MASK (PERL_COUNT_MULTIPLIER * 0x80) # define PERL_WORDSIZE sizeof(PERL_UINTMAX_T) # define PERL_WORD_BOUNDARY_MASK (PERL_WORDSIZE - 1) /* Evaluates to 0 if 'x' is at a word boundary; otherwise evaluates to 1, by * or'ing together the lowest bits of 'x'. Hopefully the final term gets * optimized out completely on a 32-bit system, and its mask gets optimized out * on a 64-bit system */ # define PERL_IS_SUBWORD_ADDR(x) (1 & ( PTR2nat(x) \ | ( PTR2nat(x) >> 1) \ | ( ( (PTR2nat(x) \ & PERL_WORD_BOUNDARY_MASK) >> 2)))) #ifndef EBCDIC /* Do the word-at-a-time iff there is at least one usable full word. That * means that after advancing to a word boundary, there still is at least a * full word left. The number of bytes needed to advance is 'wordsize - * offset' unless offset is 0. */ if ((STRLEN) (send - x) >= PERL_WORDSIZE /* This term is wordsize if subword; 0 if not */ + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(x) /* 'offset' */ - (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK)) { /* Process per-byte until reach word boundary. XXX This loop could be * eliminated if we knew that this platform had fast unaligned reads */ while (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK) { if (! UTF8_IS_INVARIANT(*x)) { if (ep) { *ep = x; } return FALSE; } x++; } /* Here, we know we have at least one full word to process. Process * per-word as long as we have at least a full word left */ do { if ((* (const PERL_UINTMAX_T *) x) & PERL_VARIANTS_WORD_MASK) { /* Found a variant. Just return if caller doesn't want its * exact position */ if (! ep) { return FALSE; } # if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 \ || BYTEORDER == 0x4321 || BYTEORDER == 0x87654321 *ep = x + variant_byte_number(* (const PERL_UINTMAX_T *) x); assert(*ep >= s && *ep < send); return FALSE; # else /* If weird byte order, drop into next loop to do byte-at-a-time checks. */ break; # endif } x += PERL_WORDSIZE; } while (x + PERL_WORDSIZE <= send); } #endif /* End of ! EBCDIC */ /* Process per-byte */ while (x < send) { if (! UTF8_IS_INVARIANT(*x)) { if (ep) { *ep = x; } return FALSE; } x++; } return TRUE; } /* See if the platform has builtins for finding the most/least significant bit, * and which one is right for using on 32 and 64 bit operands */ #if (__has_builtin(__builtin_clz) || PERL_GCC_VERSION_GE(3,4,0)) # if U32SIZE == INTSIZE # define PERL_CLZ_32 __builtin_clz # endif # if defined(U64TYPE) && U64SIZE == INTSIZE # define PERL_CLZ_64 __builtin_clz # endif #endif #if (__has_builtin(__builtin_ctz) || PERL_GCC_VERSION_GE(3,4,0)) # if U32SIZE == INTSIZE # define PERL_CTZ_32 __builtin_ctz # endif # if defined(U64TYPE) && U64SIZE == INTSIZE # define PERL_CTZ_64 __builtin_ctz # endif #endif #if (__has_builtin(__builtin_clzl) || PERL_GCC_VERSION_GE(3,4,0)) # if U32SIZE == LONGSIZE && ! defined(PERL_CLZ_32) # define PERL_CLZ_32 __builtin_clzl # endif # if defined(U64TYPE) && U64SIZE == LONGSIZE && ! defined(PERL_CLZ_64) # define PERL_CLZ_64 __builtin_clzl # endif #endif #if (__has_builtin(__builtin_ctzl) || PERL_GCC_VERSION_GE(3,4,0)) # if U32SIZE == LONGSIZE && ! defined(PERL_CTZ_32) # define PERL_CTZ_32 __builtin_ctzl # endif # if defined(U64TYPE) && U64SIZE == LONGSIZE && ! defined(PERL_CTZ_64) # define PERL_CTZ_64 __builtin_ctzl # endif #endif #if (__has_builtin(__builtin_clzll) || PERL_GCC_VERSION_GE(3,4,0)) # if U32SIZE == LONGLONGSIZE && ! defined(PERL_CLZ_32) # define PERL_CLZ_32 __builtin_clzll # endif # if defined(U64TYPE) && U64SIZE == LONGLONGSIZE && ! defined(PERL_CLZ_64) # define PERL_CLZ_64 __builtin_clzll # endif #endif #if (__has_builtin(__builtin_ctzll) || PERL_GCC_VERSION_GE(3,4,0)) # if U32SIZE == LONGLONGSIZE && ! defined(PERL_CTZ_32) # define PERL_CTZ_32 __builtin_ctzll # endif # if defined(U64TYPE) && U64SIZE == LONGLONGSIZE && ! defined(PERL_CTZ_64) # define PERL_CTZ_64 __builtin_ctzll # endif #endif #if defined(_MSC_VER) # include
# pragma intrinsic(_BitScanForward) # pragma intrinsic(_BitScanReverse) # ifdef _WIN64 # pragma intrinsic(_BitScanForward64) # pragma intrinsic(_BitScanReverse64) # endif #endif /* The reason there are not checks to see if ffs() and ffsl() are available for * determining the lsb, is because these don't improve on the deBruijn method * fallback, which is just a branchless integer multiply, array element * retrieval, and shift. The others, even if the function call overhead is * optimized out, have to cope with the possibility of the input being all * zeroes, and almost certainly will have conditionals for this eventuality. * khw, at the time of this commit, looked at the source for both gcc and clang * to verify this. (gcc used a method inferior to deBruijn.) */ /* Below are functions to find the first, last, or only set bit in a word. On * platforms with 64-bit capability, there is a pair for each operation; the * first taking a 64 bit operand, and the second a 32 bit one. The logic is * the same in each pair, so the second is stripped of most comments. */ #ifdef U64TYPE /* HAS_QUAD not usable outside the core */ PERL_STATIC_INLINE unsigned Perl_lsbit_pos64(U64 word) { /* Find the position (0..63) of the least significant set bit in the input * word */ ASSUME(word != 0); /* If we can determine that the platform has a usable fast method to get * this info, use that */ # if defined(PERL_CTZ_64) # define PERL_HAS_FAST_GET_LSB_POS64 return (unsigned) PERL_CTZ_64(word); # elif U64SIZE == 8 && defined(_WIN64) && defined(_MSC_VER) # define PERL_HAS_FAST_GET_LSB_POS64 { unsigned long index; _BitScanForward64(&index, word); return (unsigned)index; } # else /* Here, we didn't find a fast method for finding the lsb. Fall back to * making the lsb the only set bit in the word, and use our function that * works on words with a single bit set. * * Isolate the lsb; * https://stackoverflow.com/questions/757059/position-of-least-significant-bit-that-is-set * * The word will look like this, with a rightmost set bit in position 's': * ('x's are don't cares, and 'y's are their complements) * s * x..x100..00 * y..y011..11 Complement * y..y100..00 Add 1 * 0..0100..00 And with the original * * (Yes, complementing and adding 1 is just taking the negative on 2's * complement machines, but not on 1's complement ones, and some compilers * complain about negating an unsigned.) */ return single_1bit_pos64(word & (~word + 1)); # endif } # define lsbit_pos_uintmax_(word) lsbit_pos64(word) #else /* ! QUAD */ # define lsbit_pos_uintmax_(word) lsbit_pos32(word) #endif PERL_STATIC_INLINE unsigned /* Like above for 32 bit word */ Perl_lsbit_pos32(U32 word) { /* Find the position (0..31) of the least significant set bit in the input * word */ ASSUME(word != 0); #if defined(PERL_CTZ_32) # define PERL_HAS_FAST_GET_LSB_POS32 return (unsigned) PERL_CTZ_32(word); #elif U32SIZE == 4 && defined(_MSC_VER) # define PERL_HAS_FAST_GET_LSB_POS32 { unsigned long index; _BitScanForward(&index, word); return (unsigned)index; } #else return single_1bit_pos32(word & (~word + 1)); #endif } /* Convert the leading zeros count to the bit position of the first set bit. * This just subtracts from the highest position, 31 or 63. But some compilers * don't optimize this optimally, and so a bit of bit twiddling encourages them * to do the right thing. It turns out that subtracting a smaller non-negative * number 'x' from 2**n-1 for any n is the same as taking the exclusive-or of * the two numbers. To see why, first note that the sum of any number, x, and * its complement, x', is all ones. So all ones minus x is x'. Then note that * the xor of x and all ones is x'. */ #define LZC_TO_MSBIT_POS_(size, lzc) ((size##SIZE * CHARBITS - 1) ^ (lzc)) #ifdef U64TYPE /* HAS_QUAD not usable outside the core */ PERL_STATIC_INLINE unsigned Perl_msbit_pos64(U64 word) { /* Find the position (0..63) of the most significant set bit in the input * word */ ASSUME(word != 0); /* If we can determine that the platform has a usable fast method to get * this, use that */ # if defined(PERL_CLZ_64) # define PERL_HAS_FAST_GET_MSB_POS64 return (unsigned) LZC_TO_MSBIT_POS_(U64, PERL_CLZ_64(word)); # elif U64SIZE == 8 && defined(_WIN64) && defined(_MSC_VER) # define PERL_HAS_FAST_GET_MSB_POS64 { unsigned long index; _BitScanReverse64(&index, word); return (unsigned)index; } # else /* Here, we didn't find a fast method for finding the msb. Fall back to * making the msb the only set bit in the word, and use our function that * works on words with a single bit set. * * Isolate the msb; http://codeforces.com/blog/entry/10330 * * Only the most significant set bit matters. Or'ing word with its right * shift of 1 makes that bit and the next one to its right both 1. * Repeating that with the right shift of 2 makes for 4 1-bits in a row. * ... We end with the msb and all to the right being 1. */ word |= (word >> 1); word |= (word >> 2); word |= (word >> 4); word |= (word >> 8); word |= (word >> 16); word |= (word >> 32); /* Then subtracting the right shift by 1 clears all but the left-most of * the 1 bits, which is our desired result */ word -= (word >> 1); /* Now we have a single bit set */ return single_1bit_pos64(word); # endif } # define msbit_pos_uintmax_(word) msbit_pos64(word) #else /* ! QUAD */ # define msbit_pos_uintmax_(word) msbit_pos32(word) #endif PERL_STATIC_INLINE unsigned Perl_msbit_pos32(U32 word) { /* Find the position (0..31) of the most significant set bit in the input * word */ ASSUME(word != 0); #if defined(PERL_CLZ_32) # define PERL_HAS_FAST_GET_MSB_POS32 return (unsigned) LZC_TO_MSBIT_POS_(U32, PERL_CLZ_32(word)); #elif U32SIZE == 4 && defined(_MSC_VER) # define PERL_HAS_FAST_GET_MSB_POS32 { unsigned long index; _BitScanReverse(&index, word); return (unsigned)index; } #else word |= (word >> 1); word |= (word >> 2); word |= (word >> 4); word |= (word >> 8); word |= (word >> 16); word -= (word >> 1); return single_1bit_pos32(word); #endif } #if UVSIZE == U64SIZE # define msbit_pos(word) msbit_pos64(word) # define lsbit_pos(word) lsbit_pos64(word) #elif UVSIZE == U32SIZE # define msbit_pos(word) msbit_pos32(word) # define lsbit_pos(word) lsbit_pos32(word) #endif #ifdef U64TYPE /* HAS_QUAD not usable outside the core */ PERL_STATIC_INLINE unsigned Perl_single_1bit_pos64(U64 word) { /* Given a 64-bit word known to contain all zero bits except one 1 bit, * find and return the 1's position: 0..63 */ # ifdef PERL_CORE /* macro not exported */ ASSUME(isPOWER_OF_2(word)); # else ASSUME(word && (word & (word-1)) == 0); # endif /* The only set bit is both the most and least significant bit. If we have * a fast way of finding either one, use that. * * It may appear at first glance that those functions call this one, but * they don't if the corresponding #define is set */ # ifdef PERL_HAS_FAST_GET_MSB_POS64 return msbit_pos64(word); # elif defined(PERL_HAS_FAST_GET_LSB_POS64) return lsbit_pos64(word); # else /* The position of the only set bit in a word can be quickly calculated * using deBruijn sequences. See for example * https://en.wikipedia.org/wiki/De_Bruijn_sequence */ return PL_deBruijn_bitpos_tab64[(word * PERL_deBruijnMagic64_) >> PERL_deBruijnShift64_]; # endif } #endif PERL_STATIC_INLINE unsigned Perl_single_1bit_pos32(U32 word) { /* Given a 32-bit word known to contain all zero bits except one 1 bit, * find and return the 1's position: 0..31 */ #ifdef PERL_CORE /* macro not exported */ ASSUME(isPOWER_OF_2(word)); #else ASSUME(word && (word & (word-1)) == 0); #endif #ifdef PERL_HAS_FAST_GET_MSB_POS32 return msbit_pos32(word); #elif defined(PERL_HAS_FAST_GET_LSB_POS32) return lsbit_pos32(word); /* Unlikely, but possible for the platform to have a wider fast operation but * not a narrower one. But easy enough to handle the case by widening the * parameter size. (Going the other way, emulating 64 bit by two 32 bit ops * would be slower than the deBruijn method.) */ #elif defined(PERL_HAS_FAST_GET_MSB_POS64) return msbit_pos64(word); #elif defined(PERL_HAS_FAST_GET_LSB_POS64) return lsbit_pos64(word); #else return PL_deBruijn_bitpos_tab32[(word * PERL_deBruijnMagic32_) >> PERL_deBruijnShift32_]; #endif } #ifndef EBCDIC PERL_STATIC_INLINE unsigned int Perl_variant_byte_number(PERL_UINTMAX_T word) { /* This returns the position in a word (0..7) of the first variant byte in * it. This is a helper function. Note that there are no branches */ /* Get just the msb bits of each byte */ word &= PERL_VARIANTS_WORD_MASK; /* This should only be called if we know there is a variant byte in the * word */ assert(word); # if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 /* Bytes are stored like * Byte8 ... Byte2 Byte1 * 63..56...15...8 7...0 * so getting the lsb of the whole modified word is getting the msb of the * first byte that has its msb set */ word = lsbit_pos_uintmax_(word); /* Here, word contains the position 7,15,23,...55,63 of that bit. Convert * to 0..7 */ return (unsigned int) ((word + 1) >> 3) - 1; # elif BYTEORDER == 0x4321 || BYTEORDER == 0x87654321 /* Bytes are stored like * Byte1 Byte2 ... Byte8 * 63..56 55..47 ... 7...0 * so getting the msb of the whole modified word is getting the msb of the * first byte that has its msb set */ word = msbit_pos_uintmax_(word); /* Here, word contains the position 63,55,...,23,15,7 of that bit. Convert * to 0..7 */ word = ((word + 1) >> 3) - 1; /* And invert the result because of the reversed byte order on this * platform */ word = CHARBITS - word - 1; return (unsigned int) word; # else # error Unexpected byte order # endif } #endif #if defined(PERL_CORE) || defined(PERL_EXT) /* =for apidoc variant_under_utf8_count This function looks at the sequence of bytes between C
and C
, which are assumed to be encoded in ASCII/Latin1, and returns how many of them would change should the string be translated into UTF-8. Due to the nature of UTF-8, each of these would occupy two bytes instead of the single one in the input string. Thus, this function returns the precise number of bytes the string would expand by when translated to UTF-8. Unlike most of the other functions that have C
in their name, the input to this function is NOT a UTF-8-encoded string. The function name is slightly I
to emphasize this. This function is internal to Perl because khw thinks that any XS code that would want this is probably operating too close to the internals. Presenting a valid use case could change that. See also C
> and C
>, =cut */ PERL_STATIC_INLINE Size_t S_variant_under_utf8_count(const U8* const s, const U8* const e) { const U8* x = s; Size_t count = 0; PERL_ARGS_ASSERT_VARIANT_UNDER_UTF8_COUNT; # ifndef EBCDIC /* Test if the string is long enough to use word-at-a-time. (Logic is the * same as for is_utf8_invariant_string()) */ if ((STRLEN) (e - x) >= PERL_WORDSIZE + PERL_WORDSIZE * PERL_IS_SUBWORD_ADDR(x) - (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK)) { /* Process per-byte until reach word boundary. XXX This loop could be * eliminated if we knew that this platform had fast unaligned reads */ while (PTR2nat(x) & PERL_WORD_BOUNDARY_MASK) { count += ! UTF8_IS_INVARIANT(*x++); } /* Process per-word as long as we have at least a full word left */ do { /* Commit 03c1e4ab1d6ee9062fb3f94b0ba31db6698724b1 contains an explanation of how this works */ PERL_UINTMAX_T increment = ((((* (PERL_UINTMAX_T *) x) & PERL_VARIANTS_WORD_MASK) >> 7) * PERL_COUNT_MULTIPLIER) >> ((PERL_WORDSIZE - 1) * CHARBITS); count += (Size_t) increment; x += PERL_WORDSIZE; } while (x + PERL_WORDSIZE <= e); } # endif /* Process per-byte */ while (x < e) { if (! UTF8_IS_INVARIANT(*x)) { count++; } x++; } return count; } #endif /* Keep these around for these files */ #if ! defined(PERL_IN_REGEXEC_C) && ! defined(PERL_IN_UTF8_C) # undef PERL_WORDSIZE # undef PERL_COUNT_MULTIPLIER # undef PERL_WORD_BOUNDARY_MASK # undef PERL_VARIANTS_WORD_MASK #endif /* =for apidoc is_utf8_string Returns TRUE if the first C
bytes of string C
form a valid Perl-extended-UTF-8 string; returns FALSE otherwise. If C
is 0, it will be calculated using C
(which means if you use this option, that C
can't have embedded C
characters and has to have a terminating C
byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'. This function considers Perl's extended UTF-8 to be valid. That means that code points above Unicode, surrogates, and non-character code points are considered valid by this function. Use C
>, C
>, or C
> to restrict what code points are considered valid. See also C
>, C
>, C
>, C
>, C
>, C
>, C
>, =cut */ #define is_utf8_string(s, len) is_utf8_string_loclen(s, len, NULL, NULL) #if defined(PERL_CORE) || defined (PERL_EXT) /* =for apidoc is_utf8_non_invariant_string Returns TRUE if L
returns FALSE for the first C
bytes of the string C
, but they are, nonetheless, legal Perl-extended UTF-8; otherwise returns FALSE. A TRUE return means that at least one code point represented by the sequence either is a wide character not representable as a single byte, or the representation differs depending on whether the sequence is encoded in UTF-8 or not. See also C
>, C
> =cut This is commonly used to determine if a SV's UTF-8 flag should be turned on. It generally needn't be if its string is entirely UTF-8 invariant, and it shouldn't be if it otherwise contains invalid UTF-8. It is an internal function because khw thinks that XS code shouldn't be working at this low a level. A valid use case could change that. */ PERL_STATIC_INLINE bool Perl_is_utf8_non_invariant_string(const U8* const s, STRLEN len) { const U8 * first_variant; PERL_ARGS_ASSERT_IS_UTF8_NON_INVARIANT_STRING; if (is_utf8_invariant_string_loc(s, len, &first_variant)) { return FALSE; } return is_utf8_string(first_variant, len - (first_variant - s)); } #endif /* =for apidoc is_strict_utf8_string Returns TRUE if the first C
bytes of string C
form a valid UTF-8-encoded string that is fully interchangeable by any application using Unicode rules; otherwise it returns FALSE. If C
is 0, it will be calculated using C
(which means if you use this option, that C
can't have embedded C
characters and has to have a terminating C
byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'. This function returns FALSE for strings containing any code points above the Unicode max of 0x10FFFF, surrogate code points, or non-character code points. See also C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, and C
>. =cut */ #define is_strict_utf8_string(s, len) is_strict_utf8_string_loclen(s, len, NULL, NULL) /* =for apidoc is_c9strict_utf8_string Returns TRUE if the first C
bytes of string C
form a valid UTF-8-encoded string that conforms to L
; otherwise it returns FALSE. If C
is 0, it will be calculated using C
(which means if you use this option, that C
can't have embedded C
characters and has to have a terminating C
byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'. This function returns FALSE for strings containing any code points above the Unicode max of 0x10FFFF or surrogate code points, but accepts non-character code points per L
. See also C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, and C
>. =cut */ #define is_c9strict_utf8_string(s, len) is_c9strict_utf8_string_loclen(s, len, NULL, 0) /* =for apidoc is_utf8_string_flags Returns TRUE if the first C
bytes of string C
form a valid UTF-8 string, subject to the restrictions imposed by C
; returns FALSE otherwise. If C
is 0, it will be calculated using C
(which means if you use this option, that C
can't have embedded C
characters and has to have a terminating C
byte). Note that all characters being ASCII constitute 'a valid UTF-8 string'. If C
is 0, this gives the same results as C
>; if C
is C
, this gives the same results as C
>; and if C
is C
, this gives the same results as C
>. Otherwise C
may be any combination of the C
> flags understood by C
>, with the same meanings. See also C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, C
>, and C
>. =cut */ PERL_STATIC_INLINE bool Perl_is_utf8_string_flags(const U8 *s, STRLEN len, const U32 flags) { const U8 * first_variant; PERL_ARGS_ASSERT_IS_UTF8_STRING_FLAGS; assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE |UTF8_DISALLOW_PERL_EXTENDED))); if (len == 0) { len = strlen((const char *)s); } if (flags == 0) { return is_utf8_string(s, len); } if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED) == UTF8_DISALLOW_ILLEGAL_INTERCHANGE) { return is_strict_utf8_string(s, len); } if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED) == UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE) { return is_c9strict_utf8_string(s, len); } if (! is_utf8_invariant_string_loc(s, len, &first_variant)) { const U8* const send = s + len; const U8* x = first_variant; while (x < send) { STRLEN cur_len = isUTF8_CHAR_flags(x, send, flags); if (UNLIKELY(! cur_len)) { return FALSE; } x += cur_len; } } return TRUE; } /* =for apidoc is_utf8_string_loc Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer. See also C
>. =cut */ #define is_utf8_string_loc(s, len, ep) is_utf8_string_loclen(s, len, ep, 0) /* =for apidoc is_utf8_string_loclen Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer, and the number of UTF-8 encoded characters in the C
pointer. See also C
>. =cut */ PERL_STATIC_INLINE bool Perl_is_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el) { const U8 * first_variant; PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN; if (len == 0) { len = strlen((const char *) s); } if (is_utf8_invariant_string_loc(s, len, &first_variant)) { if (el) *el = len; if (ep) { *ep = s + len; } return TRUE; } { const U8* const send = s + len; const U8* x = first_variant; STRLEN outlen = first_variant - s; while (x < send) { const STRLEN cur_len = isUTF8_CHAR(x, send); if (UNLIKELY(! cur_len)) { break; } x += cur_len; outlen++; } if (el) *el = outlen; if (ep) { *ep = x; } return (x == send); } } /* The perl core arranges to never call the DFA below without there being at * least one byte available to look at. This allows the DFA to use a do {} * while loop which means that calling it with a UTF-8 invariant has a single * conditional, same as the calling code checking for invariance ahead of time. * And having the calling code remove that conditional speeds up by that * conditional, the case where it wasn't invariant. So there's no reason to * check before caling this. * * But we don't know this for non-core calls, so have to retain the check for * them. */ #ifdef PERL_CORE # define PERL_NON_CORE_CHECK_EMPTY(s,e) assert((e) > (s)) #else # define PERL_NON_CORE_CHECK_EMPTY(s,e) if ((e) <= (s)) return FALSE #endif /* * DFA for checking input is valid UTF-8 syntax. * * This uses adaptations of the table and algorithm given in * https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive * documentation of the original version. A copyright notice for the original * version is given at the beginning of this file. The Perl adaptations are * documented at the definition of PL_extended_utf8_dfa_tab[]. * * This dfa is fast. There are three exit conditions: * 1) a well-formed code point, acceptable to the table * 2) the beginning bytes of an incomplete character, whose completion might * or might not be acceptable * 3) unacceptable to the table. Some of the adaptations have certain, * hopefully less likely to occur, legal inputs be unacceptable to the * table, so these must be sorted out afterwards. * * This macro is a complete implementation of the code executing the DFA. It * is passed the input sequence bounds and the table to use, and what to do * for each of the exit conditions. There are three canned actions, likely to * be the ones you want: * DFA_RETURN_SUCCESS_ * DFA_RETURN_FAILURE_ * DFA_GOTO_TEASE_APART_FF_ * * You pass a parameter giving the action to take for each of the three * possible exit conditions: * * 'accept_action' This is executed when the DFA accepts the input. * DFA_RETURN_SUCCESS_ is the most likely candidate. * 'reject_action' This is executed when the DFA rejects the input. * DFA_RETURN_FAILURE_ is a candidate, or 'goto label' where * you have written code to distinguish the rejecting state * results. Because it happens in several places, and * involves #ifdefs, the special action * DFA_GOTO_TEASE_APART_FF_ is what you want with * PL_extended_utf8_dfa_tab. On platforms without * EXTRA_LONG_UTF8, there is no need to tease anything apart, * so this evaluates to DFA_RETURN_FAILURE_; otherwise you * need to have a label 'tease_apart_FF' that it will transfer * to. * 'incomplete_char_action' This is executed when the DFA ran off the end * before accepting or rejecting the input. * DFA_RETURN_FAILURE_ is the likely action, but you could * have a 'goto', or NOOP. In the latter case the DFA drops * off the end, and you place your code to handle this case * immediately after it. */ #define DFA_RETURN_SUCCESS_ return s - s0 #define DFA_RETURN_FAILURE_ return 0 #ifdef HAS_EXTRA_LONG_UTF8 # define DFA_TEASE_APART_FF_ goto tease_apart_FF #else # define DFA_TEASE_APART_FF_ DFA_RETURN_FAILURE_ #endif #define PERL_IS_UTF8_CHAR_DFA(s0, e, dfa_tab, \ accept_action, \ reject_action, \ incomplete_char_action) \ STMT_START { \ const U8 * s = s0; \ const U8 * e_ = e; \ UV state = 0; \ \ PERL_NON_CORE_CHECK_EMPTY(s, e_); \ \ do { \ state = dfa_tab[256 + state + dfa_tab[*s]]; \ s++; \ \ if (state == 0) { /* Accepting state */ \ accept_action; \ } \ \ if (UNLIKELY(state == 1)) { /* Rejecting state */ \ reject_action; \ } \ } while (s < e_); \ \ /* Here, dropped out of loop before end-of-char */ \ incomplete_char_action; \ } STMT_END /* =for apidoc isUTF8_CHAR Evaluates to non-zero if the first few bytes of the string starting at C
and looking no further than S
> are well-formed UTF-8, as extended by Perl, that represents some code point; otherwise it evaluates to 0. If non-zero, the value gives how many bytes starting at C
comprise the code point's representation. Any bytes remaining before C
, but beyond the ones needed to form the first code point in C
, are not examined. The code point can be any that will fit in an IV on this machine, using Perl's extension to official UTF-8 to represent those higher than the Unicode maximum of 0x10FFFF. That means that this macro is used to efficiently decide if the next few bytes in C
is legal UTF-8 for a single character. Use C
> to restrict the acceptable code points to those defined by Unicode to be fully interchangeable across applications; C
> to use the L
definition of allowable code points; and C
> for a more customized definition. Use C
>, C
>, and C
> to check entire strings. Note also that a UTF-8 "invariant" character (i.e. ASCII on non-EBCDIC machines) is a valid UTF-8 character. =cut This uses an adaptation of the table and algorithm given in https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive documentation of the original version. A copyright notice for the original version is given at the beginning of this file. The Perl adaptation is documented at the definition of PL_extended_utf8_dfa_tab[]. */ PERL_STATIC_INLINE Size_t Perl_isUTF8_CHAR(const U8 * const s0, const U8 * const e) { PERL_ARGS_ASSERT_ISUTF8_CHAR; PERL_IS_UTF8_CHAR_DFA(s0, e, PL_extended_utf8_dfa_tab, DFA_RETURN_SUCCESS_, DFA_TEASE_APART_FF_, DFA_RETURN_FAILURE_); /* Here, we didn't return success, but dropped out of the loop. In the * case of PL_extended_utf8_dfa_tab, this means the input is either * malformed, or the start byte was FF on a platform that the dfa doesn't * handle FF's. Call a helper function. */ #ifdef HAS_EXTRA_LONG_UTF8 tease_apart_FF: /* In the case of PL_extended_utf8_dfa_tab, getting here means the input is * either malformed, or was for the largest possible start byte, which we * now check, not inline */ if (*s0 != I8_TO_NATIVE_UTF8(0xFF)) { return 0; } return is_utf8_FF_helper_(s0, e, FALSE /* require full, not partial char */ ); #endif } /* =for apidoc isSTRICT_UTF8_CHAR Evaluates to non-zero if the first few bytes of the string starting at C
and looking no further than S
> are well-formed UTF-8 that represents some Unicode code point completely acceptable for open interchange between all applications; otherwise it evaluates to 0. If non-zero, the value gives how many bytes starting at C
comprise the code point's representation. Any bytes remaining before C
, but beyond the ones needed to form the first code point in C
, are not examined. The largest acceptable code point is the Unicode maximum 0x10FFFF, and must not be a surrogate nor a non-character code point. Thus this excludes any code point from Perl's extended UTF-8. This is used to efficiently decide if the next few bytes in C
is legal Unicode-acceptable UTF-8 for a single character. Use C
> to use the L
definition of allowable code points; C
> to check for Perl's extended UTF-8; and C
> for a more customized definition. Use C
>, C
>, and C
> to check entire strings. =cut This uses an adaptation of the tables and algorithm given in https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive documentation of the original version. A copyright notice for the original version is given at the beginning of this file. The Perl adaptation is documented at the definition of strict_extended_utf8_dfa_tab[]. */ PERL_STATIC_INLINE Size_t Perl_isSTRICT_UTF8_CHAR(const U8 * const s0, const U8 * const e) { PERL_ARGS_ASSERT_ISSTRICT_UTF8_CHAR; PERL_IS_UTF8_CHAR_DFA(s0, e, PL_strict_utf8_dfa_tab, DFA_RETURN_SUCCESS_, goto check_hanguls, DFA_RETURN_FAILURE_); check_hanguls: /* Here, we didn't return success, but dropped out of the loop. In the * case of PL_strict_utf8_dfa_tab, this means the input is either * malformed, or was for certain Hanguls; handle them specially */ /* The dfa above drops out for incomplete or illegal inputs, and certain * legal Hanguls; check and return accordingly */ return is_HANGUL_ED_utf8_safe(s0, e); } /* =for apidoc isC9_STRICT_UTF8_CHAR Evaluates to non-zero if the first few bytes of the string starting at C
and looking no further than S
> are well-formed UTF-8 that represents some Unicode non-surrogate code point; otherwise it evaluates to 0. If non-zero, the value gives how many bytes starting at C
comprise the code point's representation. Any bytes remaining before C
, but beyond the ones needed to form the first code point in C
, are not examined. The largest acceptable code point is the Unicode maximum 0x10FFFF. This differs from C
> only in that it accepts non-character code points. This corresponds to L
. which said that non-character code points are merely discouraged rather than completely forbidden in open interchange. See L
. Use C
> to check for Perl's extended UTF-8; and C
> for a more customized definition. Use C
>, C
>, and C
> to check entire strings. =cut This uses an adaptation of the tables and algorithm given in https://bjoern.hoehrmann.de/utf-8/decoder/dfa/, which provides comprehensive documentation of the original version. A copyright notice for the original version is given at the beginning of this file. The Perl adaptation is documented at the definition of PL_c9_utf8_dfa_tab[]. */ PERL_STATIC_INLINE Size_t Perl_isC9_STRICT_UTF8_CHAR(const U8 * const s0, const U8 * const e) { PERL_ARGS_ASSERT_ISC9_STRICT_UTF8_CHAR; PERL_IS_UTF8_CHAR_DFA(s0, e, PL_c9_utf8_dfa_tab, DFA_RETURN_SUCCESS_, DFA_RETURN_FAILURE_, DFA_RETURN_FAILURE_); } /* =for apidoc is_strict_utf8_string_loc Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer. See also C
>. =cut */ #define is_strict_utf8_string_loc(s, len, ep) \ is_strict_utf8_string_loclen(s, len, ep, 0) /* =for apidoc is_strict_utf8_string_loclen Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer, and the number of UTF-8 encoded characters in the C
pointer. See also C
>. =cut */ PERL_STATIC_INLINE bool Perl_is_strict_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el) { const U8 * first_variant; PERL_ARGS_ASSERT_IS_STRICT_UTF8_STRING_LOCLEN; if (len == 0) { len = strlen((const char *) s); } if (is_utf8_invariant_string_loc(s, len, &first_variant)) { if (el) *el = len; if (ep) { *ep = s + len; } return TRUE; } { const U8* const send = s + len; const U8* x = first_variant; STRLEN outlen = first_variant - s; while (x < send) { const STRLEN cur_len = isSTRICT_UTF8_CHAR(x, send); if (UNLIKELY(! cur_len)) { break; } x += cur_len; outlen++; } if (el) *el = outlen; if (ep) { *ep = x; } return (x == send); } } /* =for apidoc is_c9strict_utf8_string_loc Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer. See also C
>. =cut */ #define is_c9strict_utf8_string_loc(s, len, ep) \ is_c9strict_utf8_string_loclen(s, len, ep, 0) /* =for apidoc is_c9strict_utf8_string_loclen Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer, and the number of UTF-8 encoded characters in the C
pointer. See also C
>. =cut */ PERL_STATIC_INLINE bool Perl_is_c9strict_utf8_string_loclen(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el) { const U8 * first_variant; PERL_ARGS_ASSERT_IS_C9STRICT_UTF8_STRING_LOCLEN; if (len == 0) { len = strlen((const char *) s); } if (is_utf8_invariant_string_loc(s, len, &first_variant)) { if (el) *el = len; if (ep) { *ep = s + len; } return TRUE; } { const U8* const send = s + len; const U8* x = first_variant; STRLEN outlen = first_variant - s; while (x < send) { const STRLEN cur_len = isC9_STRICT_UTF8_CHAR(x, send); if (UNLIKELY(! cur_len)) { break; } x += cur_len; outlen++; } if (el) *el = outlen; if (ep) { *ep = x; } return (x == send); } } /* =for apidoc is_utf8_string_loc_flags Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer. See also C
>. =cut */ #define is_utf8_string_loc_flags(s, len, ep, flags) \ is_utf8_string_loclen_flags(s, len, ep, 0, flags) /* The above 3 actual functions could have been moved into the more general one * just below, and made #defines that call it with the right 'flags'. They are * currently kept separate to increase their chances of getting inlined */ /* =for apidoc is_utf8_string_loclen_flags Like C
> but stores the location of the failure (in the case of "utf8ness failure") or the location C
+C
(in the case of "utf8ness success") in the C
pointer, and the number of UTF-8 encoded characters in the C
pointer. See also C
>. =cut */ PERL_STATIC_INLINE bool Perl_is_utf8_string_loclen_flags(const U8 *s, STRLEN len, const U8 **ep, STRLEN *el, const U32 flags) { const U8 * first_variant; PERL_ARGS_ASSERT_IS_UTF8_STRING_LOCLEN_FLAGS; assert(0 == (flags & ~(UTF8_DISALLOW_ILLEGAL_INTERCHANGE |UTF8_DISALLOW_PERL_EXTENDED))); if (len == 0) { len = strlen((const char *) s); } if (flags == 0) { return is_utf8_string_loclen(s, len, ep, el); } if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED) == UTF8_DISALLOW_ILLEGAL_INTERCHANGE) { return is_strict_utf8_string_loclen(s, len, ep, el); } if ((flags & ~UTF8_DISALLOW_PERL_EXTENDED) == UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE) { return is_c9strict_utf8_string_loclen(s, len, ep, el); } if (is_utf8_invariant_string_loc(s, len, &first_variant)) { if (el) *el = len; if (ep) { *ep = s + len; } return TRUE; } { const U8* send = s + len; const U8* x = first_variant; STRLEN outlen = first_variant - s; while (x < send) { const STRLEN cur_len = isUTF8_CHAR_flags(x, send, flags); if (UNLIKELY(! cur_len)) { break; } x += cur_len; outlen++; } if (el) *el = outlen; if (ep) { *ep = x; } return (x == send); } } /* =for apidoc utf8_distance Returns the number of UTF-8 characters between the UTF-8 pointers C
and C
. WARNING: use only if you *know* that the pointers point inside the same UTF-8 buffer. =cut */ PERL_STATIC_INLINE IV Perl_utf8_distance(pTHX_ const U8 *a, const U8 *b) { PERL_ARGS_ASSERT_UTF8_DISTANCE; return (a < b) ? -1 * (IV) utf8_length(a, b) : (IV) utf8_length(b, a); } /* =for apidoc utf8_hop Return the UTF-8 pointer C
displaced by C
characters, either forward (if C
is positive) or backward (if negative). C
does not need to be pointing to the starting byte of a character. If it isn't, one count of C
will be used up to get to the start of the next character for forward hops, and to the start of the current character for negative ones. WARNING: Prefer L to this one. Do NOT use this function unless you B