UTF-8 helpers for decoding byte streams, encoding char32_t values, and running ASCII-style character classification on char32_t. More...

Classes
struct	Char32
	Wrapper for char32_t with an operator<< that writes UTF-8. More...

Functions
constexpr size_t	num_bytes (char8_t c) noexcept
	Returns the expected number of bytes for an UTF-8 char sequence by inspecting the first byte.
constexpr char32_t	append (char32_t c, char8_t b) noexcept
	Append `b` to `c` for converting UTF-8 to UTF-32.
constexpr char32_t	first (char32_t c, char32_t num) noexcept
	Get relevant bits of first UTF-8 byte `c` of a multi-byte sequence consisting of `num` bytes.
constexpr char32_t	min_code_point (size_t num) noexcept
	Minimum Unicode scalar value representable in an UTF-8 sequence of `num` bytes.
constexpr char8_t	is_valid234 (char8_t c) noexcept
	Is the 2nd, 3rd, or 4th byte of an UTF-8 byte sequence valid?
constexpr bool	is_scalar_value (char32_t c) noexcept
	Is `c` a valid Unicode scalar value?
char32_t	decode (std::istream &is)
	Decodes the next UTF-8 sequence from `is` into a single char32_t.
char32_t	decode (std::string_view str, size_t &i) noexcept
	Decodes the UTF-8 sequence at `i` in `str` and advances `i` past it.
size_t	num_code_points (std::string_view str) noexcept
	Number of UTF-8 code points in `str`.
bool	encode (std::ostream &os, char32_t c32)
	Encodes `c32` as UTF-8 and writes the resulting bytes to `os`.
template<class... T>
constexpr auto	any (T... args) noexcept
	Build a predicate that checks whether a code point matches any of the given values.
Character classification
char32_t-style counterparts of the <ctype> functions, for a code point of any width - everything above U+00FF belongs to no class.
constexpr bool	isascii (char32_t c) noexcept
constexpr bool	isupper (char32_t c) noexcept
constexpr bool	islower (char32_t c) noexcept
constexpr bool	isdigit (char32_t c) noexcept
constexpr bool	isalpha (char32_t c) noexcept
constexpr bool	isalnum (char32_t c) noexcept
constexpr bool	isxdigit (char32_t c) noexcept
constexpr bool	iscntrl (char32_t c) noexcept
constexpr bool	isblank (char32_t c) noexcept
constexpr bool	isspace (char32_t c) noexcept
constexpr bool	isgraph (char32_t c) noexcept
constexpr bool	isprint (char32_t c) noexcept
constexpr bool	ispunct (char32_t c) noexcept
constexpr char32_t	tolower (char32_t c) noexcept
constexpr char32_t	toupper (char32_t c) noexcept
constexpr bool	isrange (char32_t c, char32_t begin, char32_t finis) noexcept
	Is `c` within [begin, finis]?
constexpr auto	isrange (char32_t begin, char32_t finis) noexcept
constexpr bool	isodigit (char32_t c) noexcept
	Is octal digit?
constexpr bool	isbdigit (char32_t c) noexcept
	Is binary digit?

Variables
static constexpr size_t	Max = 4
	Maximal number of char8_ts of an UTF-8 byte sequence.
static constexpr char32_t	BOM = 0xfeff
	Byte Order Mark.
static constexpr std::string_view	Bom = "\xef\xbb\xbf"
	BOM as UTF-8 bytes.
static constexpr char32_t	EoF = (char32_t)std::istream::traits_type::eof()
	End of stream sentinel returned by decode.
static constexpr char32_t	Null = 0
	U+0000 NULL returned unchanged by decode.
static constexpr char32_t	Invalid = 0x110000
	Sentinel returned by decode for malformed UTF-8.

Detailed Description

UTF-8 helpers for decoding byte streams, encoding char32_t values, and running ASCII-style character classification on char32_t.

The central entry points are decode and encode. Decoding returns sentinel values such as EoF and Invalid instead of throwing.

Function Documentation

◆ any()

template<class... T>

auto fe::utf8::any ( T... args )

constexprnoexcept

Build a predicate that checks whether a code point matches any of the given values.

Definition at line 178 of file utf8.h.

References any().

Referenced by any().

◆ append()

char32_t fe::utf8::append	(	char32_t	c,
		char8_t	b )

constexprnoexcept

Append b to c for converting UTF-8 to UTF-32.

Definition at line 35 of file utf8.h.

References append().

Referenced by append(), decode(), and decode().

◆ decode() [1/2]

char32_t fe::utf8::decode ( std::istream & is )

inline

Decodes the next UTF-8 sequence from is into a single char32_t.

Returns EoF when the stream is exhausted and Invalid for malformed, overlong, surrogate, or otherwise non-scalar encodings.

Definition at line 64 of file utf8.h.

References append(), decode(), EoF, first(), Invalid, is_scalar_value(), is_valid234(), min_code_point(), and num_bytes().

Referenced by decode(), and decode().

◆ decode() [2/2]

char32_t fe::utf8::decode	(	std::string_view	str,
		size_t &	i )

inlinenoexcept

Decodes the UTF-8 sequence at i in str and advances i past it.

Returns EoF at the end of str - leaving i alone - and Invalid for malformed, overlong, surrogate, or otherwise non-scalar encodings.

Note: An Invalid sequence advances i by a single byte, so the next decode resynchronizes instead of swallowing bytes that may well start a valid sequence themselves.

Definition at line 92 of file utf8.h.

References append(), decode(), EoF, first(), Invalid, is_scalar_value(), is_valid234(), min_code_point(), and num_bytes().

◆ encode()

bool fe::utf8::encode	(	std::ostream &	os,
		char32_t	c32 )

Encodes c32 as UTF-8 and writes the resulting bytes to os.

Returns: false when c32 is outside the encodable range.

References encode().

Referenced by encode(), and fe::utf8::Char32::operator<<.

◆ first()

char32_t fe::utf8::first	(	char32_t	c,
		char32_t	num )

constexprnoexcept

Get relevant bits of first UTF-8 byte c of a multi-byte sequence consisting of num bytes.

Definition at line 38 of file utf8.h.

References first().

Referenced by decode(), decode(), and first().

◆ is_scalar_value()

bool fe::utf8::is_scalar_value ( char32_t c )

constexprnoexcept

Is c a valid Unicode scalar value?

Definition at line 58 of file utf8.h.

References is_scalar_value().

Referenced by decode(), decode(), and is_scalar_value().

◆ is_valid234()

char8_t fe::utf8::is_valid234 ( char8_t c )

constexprnoexcept

Is the 2nd, 3rd, or 4th byte of an UTF-8 byte sequence valid?

Returns: the extracted char8_t or char8_t(-1) if invalid.

Definition at line 53 of file utf8.h.

References is_valid234().

Referenced by decode(), decode(), and is_valid234().

◆ isalnum()

bool fe::utf8::isalnum ( char32_t c )

constexprnoexcept

Definition at line 147 of file utf8.h.

References isalnum(), isalpha(), and isdigit().

Referenced by isalnum(), and ispunct().

◆ isalpha()

bool fe::utf8::isalpha ( char32_t c )

constexprnoexcept

Definition at line 146 of file utf8.h.

References isalpha(), islower(), and isupper().

Referenced by isalnum(), and isalpha().

◆ isascii()

bool fe::utf8::isascii ( char32_t c )

constexprnoexcept

Definition at line 142 of file utf8.h.

References isascii().

Referenced by isascii().

◆ isbdigit()

bool fe::utf8::isbdigit ( char32_t c )

constexprnoexcept

Is binary digit?

Definition at line 163 of file utf8.h.

References isbdigit(), and isrange().

Referenced by isbdigit().

◆ isblank()

bool fe::utf8::isblank ( char32_t c )

constexprnoexcept

Definition at line 150 of file utf8.h.

References isblank().

Referenced by isblank().

◆ iscntrl()

bool fe::utf8::iscntrl ( char32_t c )

constexprnoexcept

Definition at line 149 of file utf8.h.

References iscntrl().

Referenced by iscntrl().

◆ isdigit()

bool fe::utf8::isdigit ( char32_t c )

constexprnoexcept

Definition at line 145 of file utf8.h.

References isdigit().

Referenced by isalnum(), isdigit(), and isxdigit().

◆ isgraph()

bool fe::utf8::isgraph ( char32_t c )

constexprnoexcept

Definition at line 152 of file utf8.h.

References isgraph().

Referenced by isgraph(), and ispunct().

◆ islower()

bool fe::utf8::islower ( char32_t c )

constexprnoexcept

Definition at line 144 of file utf8.h.

References islower().

Referenced by isalpha(), islower(), and toupper().

◆ isodigit()

bool fe::utf8::isodigit ( char32_t c )

constexprnoexcept

Is octal digit?

Definition at line 162 of file utf8.h.

References isodigit(), and isrange().

Referenced by isodigit().

◆ isprint()

bool fe::utf8::isprint ( char32_t c )

constexprnoexcept

Definition at line 153 of file utf8.h.

References isprint().

Referenced by isprint().

◆ ispunct()

bool fe::utf8::ispunct ( char32_t c )

constexprnoexcept

Definition at line 154 of file utf8.h.

References isalnum(), isgraph(), and ispunct().

Referenced by ispunct().

◆ isrange() [1/2]

auto fe::utf8::isrange	(	char32_t	begin,
		char32_t	finis )

constexprnoexcept

Definition at line 159 of file utf8.h.

References isrange().

◆ isrange() [2/2]

bool fe::utf8::isrange	(	char32_t	c,
		char32_t	begin,
		char32_t	finis )

constexprnoexcept

Is c within [begin, finis]?

Definition at line 158 of file utf8.h.

References isrange().

Referenced by isbdigit(), isodigit(), isrange(), and isrange().

◆ isspace()

bool fe::utf8::isspace ( char32_t c )

constexprnoexcept

Definition at line 151 of file utf8.h.

References isspace().

Referenced by isspace().

◆ isupper()

bool fe::utf8::isupper ( char32_t c )

constexprnoexcept

Definition at line 143 of file utf8.h.

References isupper().

Referenced by isalpha(), isupper(), and tolower().

◆ isxdigit()

bool fe::utf8::isxdigit ( char32_t c )

constexprnoexcept

Definition at line 148 of file utf8.h.

References isdigit(), and isxdigit().

Referenced by isxdigit().

◆ min_code_point()

char32_t fe::utf8::min_code_point ( size_t num )

constexprnoexcept

Minimum Unicode scalar value representable in an UTF-8 sequence of num bytes.

Definition at line 41 of file utf8.h.

References min_code_point().

Referenced by decode(), decode(), and min_code_point().

◆ num_bytes()

size_t fe::utf8::num_bytes ( char8_t c )

constexprnoexcept

Returns the expected number of bytes for an UTF-8 char sequence by inspecting the first byte.

Retuns 0 if invalid.

Definition at line 26 of file utf8.h.

References num_bytes().

Referenced by decode(), decode(), and num_bytes().

◆ num_code_points()

size_t fe::utf8::num_code_points ( std::string_view str )

noexcept

Number of UTF-8 code points in str.

Note: A column is counted in code points, so this is what turns a byte offset into one. Counts via decode, so a malformed sequence resynchronizes the same way the lexer does and a str truncated mid-sequence counts its final partial one.

References num_code_points().

Referenced by num_code_points().

◆ tolower()

char32_t fe::utf8::tolower ( char32_t c )

constexprnoexcept

Definition at line 155 of file utf8.h.

References isupper(), and tolower().

Referenced by fe::Lexer< K, S >::lower(), and tolower().

◆ toupper()

char32_t fe::utf8::toupper ( char32_t c )

constexprnoexcept

Definition at line 156 of file utf8.h.

References islower(), and toupper().

Referenced by toupper(), and fe::Lexer< K, S >::upper().

Variable Documentation

◆ BOM

char32_t fe::utf8::BOM = 0xfeff

staticconstexpr

Byte Order Mark.

Definition at line 17 of file utf8.h.

◆ Bom

std::string_view fe::utf8::Bom = "\xef\xbb\xbf"

staticconstexpr

BOM as UTF-8 bytes.

Definition at line 18 of file utf8.h.

◆ EoF

char32_t fe::utf8::EoF = (char32_t)std::istream::traits_type::eof()

staticconstexpr

End of stream sentinel returned by decode.

Definition at line 19 of file utf8.h.

Referenced by decode(), and decode().

◆ Invalid

char32_t fe::utf8::Invalid = 0x110000

staticconstexpr

Sentinel returned by decode for malformed UTF-8.

Definition at line 22 of file utf8.h.

Referenced by decode(), and decode().

◆ Max

size_t fe::utf8::Max = 4

staticconstexpr

Maximal number of char8_ts of an UTF-8 byte sequence.

Definition at line 16 of file utf8.h.

◆ Null

char32_t fe::utf8::Null = 0

staticconstexpr

U+0000 NULL returned unchanged by decode.

Definition at line 21 of file utf8.h.

Classes

Functions

Variables

Detailed Description

Function Documentation

◆ any()

◆ append()

◆ decode() [1/2]

◆ decode() [2/2]

◆ encode()

◆ first()

◆ is_scalar_value()

◆ is_valid234()

◆ isalnum()

◆ isalpha()

◆ isascii()

◆ isbdigit()

◆ isblank()

◆ iscntrl()

◆ isdigit()

◆ isgraph()

◆ islower()

◆ isodigit()

◆ isprint()

◆ ispunct()

◆ isrange() [1/2]

◆ isrange() [2/2]

◆ isspace()

◆ isupper()

◆ isxdigit()

◆ min_code_point()

◆ num_bytes()

◆ num_code_points()

◆ tolower()

◆ toupper()

Variable Documentation

◆ BOM

◆ Bom

◆ EoF

◆ Invalid

◆ Max

◆ Null