1 // SPDX-License-Identifier: GPL-2.0-only
26 #include "copy-unaligned.h"
28 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
32 #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)
36 /* Host ISA bitmap */
39 /* Per-cpu ISA extensions. */
46  * riscv_isa_extension_base() - Get base extension word
48  * @isa_bitmap: ISA bitmap to use
51  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
62  * __riscv_isa_extension_available() - Check whether given extension
65  * @isa_bitmap: ISA bitmap to use
69  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
87 			pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n");  in riscv_isa_extension_check()
90 			pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n");  in riscv_isa_extension_check()
96 			pr_err("Zicboz detected in ISA string, but no cboz-block-size found\n");  in riscv_isa_extension_check()
99 			pr_err("cboz-block-size present, but is not a power-of-2\n");  in riscv_isa_extension_check()
115  * The canonical order of ISA extension names in the ISA string is defined in
118  * Ordinarily, for in-kernel data structures, this order is unimportant but
119  * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo.
124  * 1. All multi-letter extensions must be separated from other extensions by an
127  * 2. Additional standard extensions (starting with 'Z') must be sorted after
128  *    single-letter extensions and before any higher-privileged extensions.
132  *    If multiple 'Z' extensions are named, they must be ordered first by
135  * 3. Standard supervisor-level extensions (starting with 'S') must be listed
136  *    after standard unprivileged extensions.  If multiple supervisor-level
137  *    extensions are listed, they must be ordered alphabetically.
139  * 4. Standard machine-level extensions (starting with 'Zxm') must be listed
140  *    after any lower-privileged, standard extensions.  If multiple
141  *    machine-level extensions are listed, they must be ordered
144  * 5. Non-standard extensions (starting with 'X') must be listed after all
145  *    standard extensions. If multiple non-standard extensions are listed, they
189 					  unsigned long *isa2hwcap, const char *isa)  in riscv_parse_isa_string()  argument
197 	isa += 4;  in riscv_parse_isa_string()
199 	while (*isa) {  in riscv_parse_isa_string()
200 		const char *ext = isa++;  in riscv_parse_isa_string()
201 		const char *ext_end = isa;  in riscv_parse_isa_string()
207 			 * Workaround for invalid single-letter 's' & 'u'(QEMU).  in riscv_parse_isa_string()
209 			 * not valid ISA extensions. It works until multi-letter  in riscv_parse_isa_string()
212 			if (ext[-1] != '_' && ext[1] == 'u') {  in riscv_parse_isa_string()
213 				++isa;  in riscv_parse_isa_string()
225 			 * As multi-letter extensions must be split from other multi-letter  in riscv_parse_isa_string()
226 			 * extensions with an "_", the end of a multi-letter extension will  in riscv_parse_isa_string()
228 			 * multi-letter extension.  in riscv_parse_isa_string()
230 			 * Next, as the extensions version is currently ignored, we  in riscv_parse_isa_string()
238 			 * character itself while eliminating the extensions version number.  in riscv_parse_isa_string()
239 			 * A simple re-increment solves this problem.  in riscv_parse_isa_string()
242 			for (; *isa && *isa != '_'; ++isa)  in riscv_parse_isa_string()
243 				if (unlikely(!isalnum(*isa)))  in riscv_parse_isa_string()
246 			ext_end = isa;  in riscv_parse_isa_string()
250 			if (!isdigit(ext_end[-1]))  in riscv_parse_isa_string()
253 			while (isdigit(*--ext_end))  in riscv_parse_isa_string()
256 			if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) {  in riscv_parse_isa_string()
261 			while (isdigit(*--ext_end))  in riscv_parse_isa_string()
268 			 * Things are a little easier for single-letter extensions, as they  in riscv_parse_isa_string()
272 			 * ensure that, when isa was incremented at the start of the loop,  in riscv_parse_isa_string()
275 			 * If we are already on a non-digit, there is nothing to do. Either  in riscv_parse_isa_string()
276 			 * we have a multi-letter extension's _, or the start of an  in riscv_parse_isa_string()
290 			if (!isdigit(*isa))  in riscv_parse_isa_string()
293 			while (isdigit(*++isa))  in riscv_parse_isa_string()
296 			if (tolower(*isa) != 'p')  in riscv_parse_isa_string()
299 			if (!isdigit(*++isa)) {  in riscv_parse_isa_string()
300 				--isa;  in riscv_parse_isa_string()
304 			while (isdigit(*++isa))  in riscv_parse_isa_string()
311 		 * The parser expects that at the start of an iteration isa points to the  in riscv_parse_isa_string()
313 		 * on meeting a non-alphanumeric character, an extra increment is needed  in riscv_parse_isa_string()
314 		 * where the succeeding extension is a multi-letter prefixed with an "_".  in riscv_parse_isa_string()
316 		if (*isa == '_')  in riscv_parse_isa_string()
317 			++isa;  in riscv_parse_isa_string()
321 			if ((ext_end - ext == strlen(name)) &&			\  in riscv_parse_isa_string()
324 				set_bit(bit, isainfo->isa);			\  in riscv_parse_isa_string()
330 			int nr = tolower(*ext) - 'a';  in riscv_parse_isa_string()
334 				set_bit(nr, isainfo->isa);  in riscv_parse_isa_string()
348 	const char *isa;  in riscv_fill_hwcap_from_isa_string()  local
371 			rc = of_property_read_string(node, "riscv,isa", &isa);  in riscv_fill_hwcap_from_isa_string()
374 				pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");  in riscv_fill_hwcap_from_isa_string()
378 			rc = acpi_get_riscv_isa(rhct, cpu, &isa);  in riscv_fill_hwcap_from_isa_string()
380 				pr_warn("Unable to get ISA for the hart - %d\n", cpu);  in riscv_fill_hwcap_from_isa_string()
385 		riscv_parse_isa_string(&this_hwcap, isainfo, isa2hwcap, isa);  in riscv_fill_hwcap_from_isa_string()
388 		 * These ones were as they were part of the base ISA when the  in riscv_fill_hwcap_from_isa_string()
389 		 * port & dt-bindings were upstreamed, and so can be set  in riscv_fill_hwcap_from_isa_string()
390 		 * unconditionally where `i` is in riscv,isa on DT systems.  in riscv_fill_hwcap_from_isa_string()
393 			set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
394 			set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
395 			set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
396 			set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa);  in riscv_fill_hwcap_from_isa_string()
400 		 * All "okay" hart should have same isa. Set HWCAP based on  in riscv_fill_hwcap_from_isa_string()
410 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_isa_string()
412 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_isa_string()
434 		if (!of_property_present(cpu_node, "riscv,isa-extensions")) {  in riscv_fill_hwcap_from_ext_list()
440 			if (of_property_match_string(cpu_node, "riscv,isa-extensions",  in riscv_fill_hwcap_from_ext_list()
447 			/* Only single letter extensions get set in hwcap */  in riscv_fill_hwcap_from_ext_list()
451 			set_bit(riscv_isa_ext[i].id, isainfo->isa);  in riscv_fill_hwcap_from_ext_list()
457 		 * All "okay" harts should have same isa. Set HWCAP based on  in riscv_fill_hwcap_from_ext_list()
466 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_ext_list()
468 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_ext_list()
472 		return -ENOENT;  in riscv_fill_hwcap_from_ext_list()
495 	isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;  in riscv_fill_hwcap()
496 	isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;  in riscv_fill_hwcap()
497 	isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A;  in riscv_fill_hwcap()
498 	isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;  in riscv_fill_hwcap()
499 	isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;  in riscv_fill_hwcap()
500 	isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;  in riscv_fill_hwcap()
501 	isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V;  in riscv_fill_hwcap()
509 			pr_info("Falling back to deprecated \"riscv,isa\"\n");  in riscv_fill_hwcap()
526 		 * ISA string in device tree might have 'v' flag, but  in riscv_fill_hwcap()
538 	pr_info("riscv: base ISA extensions %s\n", print_str);  in riscv_fill_hwcap()
551 	hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1));  in riscv_get_elf_hwcap()
582 	word_cycles = -1ULL;  in check_unaligned_access()
602 		if ((end_cycles - start_cycles) < word_cycles)  in check_unaligned_access()
603 			word_cycles = end_cycles - start_cycles;  in check_unaligned_access()
606 	byte_cycles = -1ULL;  in check_unaligned_access()
618 		if ((end_cycles - start_cycles) < byte_cycles)  in check_unaligned_access()
619 			byte_cycles = end_cycles - start_cycles;  in check_unaligned_access()
660  * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the
661  * patch site is for an erratum, identified by the 32-bit patch ID. When
663  * further break down patch ID into two 16-bit numbers. The lower 16 bits
667  * patching on a per-site basis will provide non-zero values and implement
702 		if (alt->vendor_id != 0)  in riscv_cpufeature_patch_func()
705 		id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);  in riscv_cpufeature_patch_func()
708 			WARN(1, "This extension id:%d is not in ISA extension list", id);  in riscv_cpufeature_patch_func()
715 		value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);  in riscv_cpufeature_patch_func()
723 		patch_text_nosync(oldptr, altptr, alt->alt_len);  in riscv_cpufeature_patch_func()
724 		riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);  in riscv_cpufeature_patch_func()