Super User's BSD Cross Reference: /NetBSD/sys/kern/subr

1 /* $NetBSD: subr_asan.c,v 1.28 2023年04月09日 09:18:09 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 *
7 * This code is part of the KASAN subsystem of the NetBSD kernel.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
32 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.28 2023年04月09日 09:18:09 riastradh Exp $");
33
34#include <sys/param.h>
35#include <sys/device.h>
36#include <sys/kernel.h>
37#include <sys/param.h>
38#include <sys/conf.h>
39#include <sys/systm.h>
40#include <sys/types.h>
41#include <sys/asan.h>
42
43#include <uvm/uvm_extern.h>
44
45#ifdef DDB
46#include <machine/db_machdep.h>
47#include <ddb/db_extern.h>
48#endif
49
50#ifdef KASAN_PANIC
51#define REPORT panic
52#else
53#define REPORT printf
54#endif
55
56 /* ASAN constants. Part of the compiler ABI. */
57#define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
58#define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1)
59#define KASAN_ALLOCA_SCALE_SIZE 32
60
61 /* The MD code. */
62#include <machine/asan.h>
63
64 /* ASAN ABI version. */
65#if defined(__clang__) && (__clang_major__ - 0 >= 6)
66#define ASAN_ABI_VERSION 8
67#elif __GNUC_PREREQ__(7, 1) && !defined(__clang__)
68#define ASAN_ABI_VERSION 8
69#elif __GNUC_PREREQ__(6, 1) && !defined(__clang__)
70#define ASAN_ABI_VERSION 6
71#else
72#error "Unsupported compiler version"
73#endif
74
75#define __RET_ADDR (unsigned long)__builtin_return_address(0)
76
77 /* Global variable descriptor. Part of the compiler ABI. */
78 struct __asan_global_source_location {
79 const char *filename;
80 int line_no;
81 int column_no;
82};
83 struct __asan_global {
84 const void *beg; /* address of the global variable */
85 size_t size; /* size of the global variable */
86 size_t size_with_redzone; /* size with the redzone */
87 const void *name; /* name of the variable */
88 const void *module_name; /* name of the module where the var is declared */
89 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */
90 struct __asan_global_source_location *location;
91#if ASAN_ABI_VERSION >= 7
92 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */
93#endif
94};
95
96 static bool kasan_enabled __read_mostly = false;
97
98 /* -------------------------------------------------------------------------- */
99
100 void
101 kasan_shadow_map(void *addr, size_t size)
102{
103 size_t sz, npages, i;
104 vaddr_t sva, eva;
105
106 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
107
108 sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE;
109
110 sva = (vaddr_t)kasan_md_addr_to_shad(addr);
111 eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz;
112
113 sva = rounddown(sva, PAGE_SIZE);
114 eva = roundup(eva, PAGE_SIZE);
115
116 npages = (eva - sva) / PAGE_SIZE;
117
118 KASSERT(sva >= KASAN_MD_SHADOW_START);
119 KASSERT(eva < KASAN_MD_SHADOW_END);
120
121 for (i = 0; i < npages; i++) {
122 kasan_md_shadow_map_page(sva + i * PAGE_SIZE);
123 }
124}
125
126 static void
127 kasan_ctors(void)
128{
129 extern Elf_Addr __CTOR_LIST__, __CTOR_END__;
130 size_t nentries, i;
131 Elf_Addr *ptr;
132
133 nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) /
134 sizeof(uintptr_t);
135
136 ptr = &__CTOR_LIST__;
137 for (i = 0; i < nentries; i++) {
138 void (*func)(void);
139
140 func = (void *)(*ptr);
141 (*func)();
142
143 ptr++;
144 }
145}
146
147 void
148 kasan_early_init(void *stack)
149{
150 kasan_md_early_init(stack);
151}
152
153 void
154 kasan_init(void)
155{
156 /* MD initialization. */
157 kasan_md_init();
158
159 /* Now officially enabled. */
160 kasan_enabled = true;
161
162 /* Call the ASAN constructors. */
163 kasan_ctors();
164}
165
166 static inline const char *
167 kasan_code_name(uint8_t code)
168{
169 switch (code) {
170 case KASAN_GENERIC_REDZONE:
171 return "GenericRedZone";
172 case KASAN_MALLOC_REDZONE:
173 return "MallocRedZone";
174 case KASAN_KMEM_REDZONE:
175 return "KmemRedZone";
176 case KASAN_POOL_REDZONE:
177 return "PoolRedZone";
178 case KASAN_POOL_FREED:
179 return "PoolUseAfterFree";
180 case 1 ... 7:
181 return "RedZonePartial";
182 case KASAN_STACK_LEFT:
183 return "StackLeft";
184 case KASAN_STACK_MID:
185 return "StackMiddle";
186 case KASAN_STACK_RIGHT:
187 return "StackRight";
188 case KASAN_USE_AFTER_RET:
189 return "UseAfterRet";
190 case KASAN_USE_AFTER_SCOPE:
191 return "UseAfterScope";
192 default:
193 return "Unknown";
194 }
195}
196
197 static void
198 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc,
199 uint8_t code)
200{
201 REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s,"
202 " %s]\n",
203 (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""),
204 (write ? "write" : "read"), kasan_code_name(code));
205 kasan_md_unwind();
206}
207
208 static __always_inline void
209 kasan_shadow_1byte_markvalid(unsigned long addr)
210{
211 int8_t *byte = kasan_md_addr_to_shad((void *)addr);
212 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
213
214 *byte = last;
215}
216
217 static __always_inline void
218 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size)
219{
220 size_t i;
221
222 for (i = 0; i < size; i++) {
223 kasan_shadow_1byte_markvalid((unsigned long)addr+i);
224 }
225}
226
227 static __always_inline void
228 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code)
229{
230 void *shad;
231
232 if (__predict_false(size == 0))
233 return;
234 if (__predict_false(kasan_md_unsupported((vaddr_t)addr)))
235 return;
236
237 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
238 KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0);
239
240 shad = (void *)kasan_md_addr_to_shad(addr);
241 size = size >> KASAN_SHADOW_SCALE_SHIFT;
242
243 __builtin_memset(shad, code, size);
244}
245
246 void
247 kasan_add_redzone(size_t *size)
248{
249 *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE);
250 *size += KASAN_SHADOW_SCALE_SIZE;
251}
252
253 void
254 kasan_softint(struct lwp *l)
255{
256 const void *stk = (const void *)uvm_lwp_getuarea(l);
257
258 kasan_shadow_Nbyte_fill(stk, USPACE, 0);
259}
260
261 /*
262 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid,
263 * and the rest as invalid. There are generally two use cases:
264 *
265 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks
266 * the redzone at the end of the buffer as invalid.
267 *
268 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid.
269 */
270 void
271 kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code)
272{
273 size_t i, n, redz;
274 int8_t *shad;
275
276 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0);
277 redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE);
278 KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0);
279 shad = kasan_md_addr_to_shad(addr);
280
281 /* Chunks of 8 bytes, valid. */
282 n = size / KASAN_SHADOW_SCALE_SIZE;
283 for (i = 0; i < n; i++) {
284 *shad++ = 0;
285 }
286
287 /* Possibly one chunk, mid. */
288 if ((size & KASAN_SHADOW_MASK) != 0) {
289 *shad++ = (size & KASAN_SHADOW_MASK);
290 }
291
292 /* Chunks of 8 bytes, invalid. */
293 n = redz / KASAN_SHADOW_SCALE_SIZE;
294 for (i = 0; i < n; i++) {
295 *shad++ = code;
296 }
297}
298
299 /* -------------------------------------------------------------------------- */
300
301#define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \
302 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \
303 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT)
304
305 static __always_inline bool
306 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code)
307{
308 int8_t *byte = kasan_md_addr_to_shad((void *)addr);
309 int8_t last = (addr & KASAN_SHADOW_MASK) + 1;
310
311 if (__predict_true(*byte == 0 || last <= *byte)) {
312 return true;
313 }
314 *code = *byte;
315 return false;
316}
317
318 static __always_inline bool
319 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code)
320{
321 int8_t *byte, last;
322
323 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) {
324 return (kasan_shadow_1byte_isvalid(addr, code) &&
325 kasan_shadow_1byte_isvalid(addr+1, code));
326 }
327
328 byte = kasan_md_addr_to_shad((void *)addr);
329 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1;
330
331 if (__predict_true(*byte == 0 || last <= *byte)) {
332 return true;
333 }
334 *code = *byte;
335 return false;
336}
337
338 static __always_inline bool
339 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code)
340{
341 int8_t *byte, last;
342
343 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) {
344 return (kasan_shadow_2byte_isvalid(addr, code) &&
345 kasan_shadow_2byte_isvalid(addr+2, code));
346 }
347
348 byte = kasan_md_addr_to_shad((void *)addr);
349 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1;
350
351 if (__predict_true(*byte == 0 || last <= *byte)) {
352 return true;
353 }
354 *code = *byte;
355 return false;
356}
357
358 static __always_inline bool
359 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code)
360{
361 int8_t *byte, last;
362
363 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) {
364 return (kasan_shadow_4byte_isvalid(addr, code) &&
365 kasan_shadow_4byte_isvalid(addr+4, code));
366 }
367
368 byte = kasan_md_addr_to_shad((void *)addr);
369 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1;
370
371 if (__predict_true(*byte == 0 || last <= *byte)) {
372 return true;
373 }
374 *code = *byte;
375 return false;
376}
377
378 static __always_inline bool
379 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code)
380{
381 size_t i;
382
383 for (i = 0; i < size; i++) {
384 if (!kasan_shadow_1byte_isvalid(addr+i, code))
385 return false;
386 }
387
388 return true;
389}
390
391 static __always_inline void
392 kasan_shadow_check(unsigned long addr, size_t size, bool write,
393 unsigned long retaddr)
394{
395 uint8_t code;
396 bool valid;
397
398 if (__predict_false(!kasan_enabled))
399 return;
400#ifdef DDB
401 if (__predict_false(db_recover != NULL))
402 return;
403#endif
404 if (__predict_false(size == 0))
405 return;
406 if (__predict_false(kasan_md_unsupported(addr)))
407 return;
408
409 if (__builtin_constant_p(size)) {
410 switch (size) {
411 case 1:
412 valid = kasan_shadow_1byte_isvalid(addr, &code);
413 break;
414 case 2:
415 valid = kasan_shadow_2byte_isvalid(addr, &code);
416 break;
417 case 4:
418 valid = kasan_shadow_4byte_isvalid(addr, &code);
419 break;
420 case 8:
421 valid = kasan_shadow_8byte_isvalid(addr, &code);
422 break;
423 default:
424 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
425 break;
426 }
427 } else {
428 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code);
429 }
430
431 if (__predict_false(!valid)) {
432 kasan_report(addr, size, write, retaddr, code);
433 }
434}
435
436 /* -------------------------------------------------------------------------- */
437
438 void *
439 kasan_memcpy(void *dst, const void *src, size_t len)
440{
441 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
442 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
443 return __builtin_memcpy(dst, src, len);
444}
445
446 int
447 kasan_memcmp(const void *b1, const void *b2, size_t len)
448{
449 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR);
450 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR);
451 return __builtin_memcmp(b1, b2, len);
452}
453
454 void *
455 kasan_memset(void *b, int c, size_t len)
456{
457 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR);
458 return __builtin_memset(b, c, len);
459}
460
461 void *
462 kasan_memmove(void *dst, const void *src, size_t len)
463{
464 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
465 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
466 return __builtin_memmove(dst, src, len);
467}
468
469 char *
470 kasan_strcpy(char *dst, const char *src)
471{
472 char *save = dst;
473
474 while (1) {
475 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR);
476 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR);
477 *dst = *src;
478 if (*src == '0円')
479 break;
480 src++, dst++;
481 }
482
483 return save;
484}
485
486 int
487 kasan_strcmp(const char *s1, const char *s2)
488{
489 while (1) {
490 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR);
491 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR);
492 if (*s1 != *s2)
493 break;
494 if (*s1 == '0円')
495 return 0;
496 s1++, s2++;
497 }
498
499 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
500}
501
502 size_t
503 kasan_strlen(const char *str)
504{
505 const char *s;
506
507 s = str;
508 while (1) {
509 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR);
510 if (*s == '0円')
511 break;
512 s++;
513 }
514
515 return (s - str);
516}
517
518 char *
519 kasan_strcat(char *dst, const char *src)
520{
521 size_t ldst, lsrc;
522
523 ldst = __builtin_strlen(dst);
524 lsrc = __builtin_strlen(src);
525 kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true,
526 __RET_ADDR);
527 kasan_shadow_check((unsigned long)src, lsrc + 1, false,
528 __RET_ADDR);
529
530 return __builtin_strcat(dst, src);
531}
532
533 char *
534 kasan_strchr(const char *s, int c)
535{
536 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
537 __RET_ADDR);
538 return __builtin_strchr(s, c);
539}
540
541 char *
542 kasan_strrchr(const char *s, int c)
543{
544 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false,
545 __RET_ADDR);
546 return __builtin_strrchr(s, c);
547}
548
549#undef kcopy
550#undef copyinstr
551#undef copyoutstr
552#undef copyin
553
554 int kasan_kcopy(const void *, void *, size_t);
555 int kasan_copyinstr(const void *, void *, size_t, size_t *);
556 int kasan_copyoutstr(const void *, void *, size_t, size_t *);
557 int kasan_copyin(const void *, void *, size_t);
558 int kcopy(const void *, void *, size_t);
559 int copyinstr(const void *, void *, size_t, size_t *);
560 int copyoutstr(const void *, void *, size_t, size_t *);
561 int copyin(const void *, void *, size_t);
562
563 int
564 kasan_kcopy(const void *src, void *dst, size_t len)
565{
566 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR);
567 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR);
568 return kcopy(src, dst, len);
569}
570
571 int
572 kasan_copyin(const void *uaddr, void *kaddr, size_t len)
573{
574 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
575 return copyin(uaddr, kaddr, len);
576}
577
578 int
579 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
580{
581 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR);
582 return copyinstr(uaddr, kaddr, len, done);
583}
584
585 int
586 kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done)
587{
588 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR);
589 return copyoutstr(kaddr, uaddr, len, done);
590}
591
592 /* -------------------------------------------------------------------------- */
593
594#undef _ucas_32
595#undef _ucas_32_mp
596#undef _ucas_64
597#undef _ucas_64_mp
598#undef _ufetch_8
599#undef _ufetch_16
600#undef _ufetch_32
601#undef _ufetch_64
602
603 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
604 int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
605 int
606 kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
607 uint32_t *ret)
608{
609 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
610 __RET_ADDR);
611 return _ucas_32(uaddr, old, new, ret);
612}
613
614#ifdef __HAVE_UCAS_MP
615 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
616 int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *);
617 int
618 kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new,
619 uint32_t *ret)
620{
621 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
622 __RET_ADDR);
623 return _ucas_32_mp(uaddr, old, new, ret);
624}
625#endif
626
627#ifdef _LP64
628 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
629 int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
630 int
631 kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
632 uint64_t *ret)
633{
634 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
635 __RET_ADDR);
636 return _ucas_64(uaddr, old, new, ret);
637}
638
639#ifdef __HAVE_UCAS_MP
640 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
641 int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *);
642 int
643 kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new,
644 uint64_t *ret)
645{
646 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true,
647 __RET_ADDR);
648 return _ucas_64_mp(uaddr, old, new, ret);
649}
650#endif
651#endif
652
653 int _ufetch_8(const uint8_t *, uint8_t *);
654 int kasan__ufetch_8(const uint8_t *, uint8_t *);
655 int
656 kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp)
657{
658 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
659 __RET_ADDR);
660 return _ufetch_8(uaddr, valp);
661}
662
663 int _ufetch_16(const uint16_t *, uint16_t *);
664 int kasan__ufetch_16(const uint16_t *, uint16_t *);
665 int
666 kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp)
667{
668 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
669 __RET_ADDR);
670 return _ufetch_16(uaddr, valp);
671}
672
673 int _ufetch_32(const uint32_t *, uint32_t *);
674 int kasan__ufetch_32(const uint32_t *, uint32_t *);
675 int
676 kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp)
677{
678 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
679 __RET_ADDR);
680 return _ufetch_32(uaddr, valp);
681}
682
683#ifdef _LP64
684 int _ufetch_64(const uint64_t *, uint64_t *);
685 int kasan__ufetch_64(const uint64_t *, uint64_t *);
686 int
687 kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp)
688{
689 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true,
690 __RET_ADDR);
691 return _ufetch_64(uaddr, valp);
692}
693#endif
694
695 /* -------------------------------------------------------------------------- */
696
697#undef atomic_add_32
698#undef atomic_add_int
699#undef atomic_add_long
700#undef atomic_add_ptr
701#undef atomic_add_64
702#undef atomic_add_32_nv
703#undef atomic_add_int_nv
704#undef atomic_add_long_nv
705#undef atomic_add_ptr_nv
706#undef atomic_add_64_nv
707#undef atomic_and_32
708#undef atomic_and_uint
709#undef atomic_and_ulong
710#undef atomic_and_64
711#undef atomic_and_32_nv
712#undef atomic_and_uint_nv
713#undef atomic_and_ulong_nv
714#undef atomic_and_64_nv
715#undef atomic_or_32
716#undef atomic_or_uint
717#undef atomic_or_ulong
718#undef atomic_or_64
719#undef atomic_or_32_nv
720#undef atomic_or_uint_nv
721#undef atomic_or_ulong_nv
722#undef atomic_or_64_nv
723#undef atomic_cas_32
724#undef atomic_cas_uint
725#undef atomic_cas_ulong
726#undef atomic_cas_ptr
727#undef atomic_cas_64
728#undef atomic_cas_32_ni
729#undef atomic_cas_uint_ni
730#undef atomic_cas_ulong_ni
731#undef atomic_cas_ptr_ni
732#undef atomic_cas_64_ni
733#undef atomic_swap_32
734#undef atomic_swap_uint
735#undef atomic_swap_ulong
736#undef atomic_swap_ptr
737#undef atomic_swap_64
738#undef atomic_dec_32
739#undef atomic_dec_uint
740#undef atomic_dec_ulong
741#undef atomic_dec_ptr
742#undef atomic_dec_64
743#undef atomic_dec_32_nv
744#undef atomic_dec_uint_nv
745#undef atomic_dec_ulong_nv
746#undef atomic_dec_ptr_nv
747#undef atomic_dec_64_nv
748#undef atomic_inc_32
749#undef atomic_inc_uint
750#undef atomic_inc_ulong
751#undef atomic_inc_ptr
752#undef atomic_inc_64
753#undef atomic_inc_32_nv
754#undef atomic_inc_uint_nv
755#undef atomic_inc_ulong_nv
756#undef atomic_inc_ptr_nv
757#undef atomic_inc_64_nv
758
759#define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \
760 void atomic_add_##name(volatile targ1 *, targ2); \
761 void kasan_atomic_add_##name(volatile targ1 *, targ2); \
762 void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \
763 { \
764 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
765 __RET_ADDR); \
766 atomic_add_##name(ptr, val); \
767 } \
768 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \
769 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \
770 tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \
771 { \
772 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
773 __RET_ADDR); \
774 return atomic_add_##name##_nv(ptr, val); \
775 }
776
777#define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \
778 void atomic_and_##name(volatile targ1 *, targ2); \
779 void kasan_atomic_and_##name(volatile targ1 *, targ2); \
780 void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \
781 { \
782 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
783 __RET_ADDR); \
784 atomic_and_##name(ptr, val); \
785 } \
786 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \
787 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \
788 tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \
789 { \
790 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
791 __RET_ADDR); \
792 return atomic_and_##name##_nv(ptr, val); \
793 }
794
795#define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \
796 void atomic_or_##name(volatile targ1 *, targ2); \
797 void kasan_atomic_or_##name(volatile targ1 *, targ2); \
798 void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \
799 { \
800 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
801 __RET_ADDR); \
802 atomic_or_##name(ptr, val); \
803 } \
804 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \
805 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \
806 tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \
807 { \
808 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
809 __RET_ADDR); \
810 return atomic_or_##name##_nv(ptr, val); \
811 }
812
813#define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \
814 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \
815 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \
816 tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \
817 { \
818 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
819 __RET_ADDR); \
820 return atomic_cas_##name(ptr, exp, new); \
821 } \
822 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
823 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \
824 tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \
825 { \
826 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
827 __RET_ADDR); \
828 return atomic_cas_##name##_ni(ptr, exp, new); \
829 }
830
831#define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \
832 tret atomic_swap_##name(volatile targ1 *, targ2); \
833 tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \
834 tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \
835 { \
836 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
837 __RET_ADDR); \
838 return atomic_swap_##name(ptr, val); \
839 }
840
841#define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \
842 void atomic_dec_##name(volatile targ1 *); \
843 void kasan_atomic_dec_##name(volatile targ1 *); \
844 void kasan_atomic_dec_##name(volatile targ1 *ptr) \
845 { \
846 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
847 __RET_ADDR); \
848 atomic_dec_##name(ptr); \
849 } \
850 tret atomic_dec_##name##_nv(volatile targ1 *); \
851 tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \
852 tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \
853 { \
854 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
855 __RET_ADDR); \
856 return atomic_dec_##name##_nv(ptr); \
857 }
858
859#define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \
860 void atomic_inc_##name(volatile targ1 *); \
861 void kasan_atomic_inc_##name(volatile targ1 *); \
862 void kasan_atomic_inc_##name(volatile targ1 *ptr) \
863 { \
864 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
865 __RET_ADDR); \
866 atomic_inc_##name(ptr); \
867 } \
868 tret atomic_inc_##name##_nv(volatile targ1 *); \
869 tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \
870 tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \
871 { \
872 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \
873 __RET_ADDR); \
874 return atomic_inc_##name##_nv(ptr); \
875 }
876
877 ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t);
878 ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t);
879 ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int);
880 ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long);
881 ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t);
882
883 ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t);
884 ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t);
885 ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int);
886 ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long);
887
888 ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t);
889 ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t);
890 ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int);
891 ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long);
892
893 ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t);
894 ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t);
895 ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int);
896 ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long);
897 ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *);
898
899 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t);
900 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t);
901 ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int);
902 ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long);
903 ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *);
904
905 ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t)
906 ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t)
907 ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int);
908 ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long);
909 ASAN_ATOMIC_FUNC_DEC(ptr, void *, void);
910
911 ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t)
912 ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t)
913 ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int);
914 ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long);
915 ASAN_ATOMIC_FUNC_INC(ptr, void *, void);
916
917 /* -------------------------------------------------------------------------- */
918
919#ifdef __HAVE_KASAN_INSTR_BUS
920
921#include <sys/bus.h>
922
923#undef bus_space_read_multi_1
924#undef bus_space_read_multi_2
925#undef bus_space_read_multi_4
926#undef bus_space_read_multi_8
927#undef bus_space_read_multi_stream_1
928#undef bus_space_read_multi_stream_2
929#undef bus_space_read_multi_stream_4
930#undef bus_space_read_multi_stream_8
931#undef bus_space_read_region_1
932#undef bus_space_read_region_2
933#undef bus_space_read_region_4
934#undef bus_space_read_region_8
935#undef bus_space_read_region_stream_1
936#undef bus_space_read_region_stream_2
937#undef bus_space_read_region_stream_4
938#undef bus_space_read_region_stream_8
939#undef bus_space_write_multi_1
940#undef bus_space_write_multi_2
941#undef bus_space_write_multi_4
942#undef bus_space_write_multi_8
943#undef bus_space_write_multi_stream_1
944#undef bus_space_write_multi_stream_2
945#undef bus_space_write_multi_stream_4
946#undef bus_space_write_multi_stream_8
947#undef bus_space_write_region_1
948#undef bus_space_write_region_2
949#undef bus_space_write_region_4
950#undef bus_space_write_region_8
951#undef bus_space_write_region_stream_1
952#undef bus_space_write_region_stream_2
953#undef bus_space_write_region_stream_4
954#undef bus_space_write_region_stream_8
955
956#define ASAN_BUS_READ_FUNC(bytes, bits) \
957 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
958 bus_size_t, uint##bits##_t *, bus_size_t); \
959 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \
960 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
961 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \
962 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
963 bus_size_t count) \
964 { \
965 kasan_shadow_check((uintptr_t)buf, \
966 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
967 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \
968 } \
969 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
970 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
971 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \
972 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
973 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \
974 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
975 bus_size_t count) \
976 { \
977 kasan_shadow_check((uintptr_t)buf, \
978 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
979 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\
980 } \
981 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \
982 bus_size_t, uint##bits##_t *, bus_size_t); \
983 void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \
984 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
985 void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \
986 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
987 bus_size_t count) \
988 { \
989 kasan_shadow_check((uintptr_t)buf, \
990 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
991 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \
992 } \
993 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \
994 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
995 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \
996 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \
997 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \
998 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \
999 bus_size_t count) \
1000 { \
1001 kasan_shadow_check((uintptr_t)buf, \
1002 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \
1003 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\
1004 }
1005
1006#define ASAN_BUS_WRITE_FUNC(bytes, bits) \
1007 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \
1008 bus_size_t, const uint##bits##_t *, bus_size_t); \
1009 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \
1010 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1011 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \
1012 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1013 bus_size_t count) \
1014 { \
1015 kasan_shadow_check((uintptr_t)buf, \
1016 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1017 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \
1018 } \
1019 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1020 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1021 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \
1022 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1023 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \
1024 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1025 bus_size_t count) \
1026 { \
1027 kasan_shadow_check((uintptr_t)buf, \
1028 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1029 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\
1030 } \
1031 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\
1032 bus_size_t, const uint##bits##_t *, bus_size_t); \
1033 void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \
1034 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1035 void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \
1036 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1037 bus_size_t count) \
1038 { \
1039 kasan_shadow_check((uintptr_t)buf, \
1040 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1041 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \
1042 } \
1043 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1044 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1045 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \
1046 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\
1047 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \
1048 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \
1049 bus_size_t count) \
1050 { \
1051 kasan_shadow_check((uintptr_t)buf, \
1052 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \
1053 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\
1054 }
1055
1056 ASAN_BUS_READ_FUNC(1, 8)
1057 ASAN_BUS_READ_FUNC(2, 16)
1058 ASAN_BUS_READ_FUNC(4, 32)
1059 ASAN_BUS_READ_FUNC(8, 64)
1060
1061 ASAN_BUS_WRITE_FUNC(1, 8)
1062 ASAN_BUS_WRITE_FUNC(2, 16)
1063 ASAN_BUS_WRITE_FUNC(4, 32)
1064 ASAN_BUS_WRITE_FUNC(8, 64)
1065
1066#endif /* __HAVE_KASAN_INSTR_BUS */
1067
1068 /* -------------------------------------------------------------------------- */
1069
1070#include <sys/mbuf.h>
1071
1072 static void
1073 kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len,
1074 bool write, uintptr_t pc)
1075{
1076 kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc);
1077}
1078
1079 static void
1080 kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len,
1081 bool write, uintptr_t pc)
1082{
1083 bus_addr_t minlen;
1084
1085 for (; m != NULL && len != 0; m = m->m_next) {
1086 kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc);
1087
1088 if (offset >= m->m_len) {
1089 offset -= m->m_len;
1090 continue;
1091 }
1092
1093 minlen = MIN(len, m->m_len - offset);
1094 kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset),
1095 minlen, write, pc);
1096
1097 offset = 0;
1098 len -= minlen;
1099 }
1100}
1101
1102 static void
1103 kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len,
1104 bool write, uintptr_t pc)
1105{
1106 bus_size_t minlen, resid;
1107 struct iovec *iov;
1108 int i;
1109
1110 kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc);
1111
1112 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace))
1113 return;
1114
1115 resid = uio->uio_resid;
1116 iov = uio->uio_iov;
1117
1118 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {
1119 kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]),
1120 false, pc);
1121 minlen = MIN(resid, iov[i].iov_len);
1122 kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen,
1123 write, pc);
1124 resid -= minlen;
1125 }
1126}
1127
1128 void
1129 kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
1130{
1131 bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0;
1132
1133 switch (map->dm_buftype) {
1134 case KASAN_DMA_LINEAR:
1135 kasan_dma_sync_linear(map->dm_buf, offset, len, write,
1136 __RET_ADDR);
1137 break;
1138 case KASAN_DMA_MBUF:
1139 kasan_dma_sync_mbuf(map->dm_buf, offset, len, write,
1140 __RET_ADDR);
1141 break;
1142 case KASAN_DMA_UIO:
1143 kasan_dma_sync_uio(map->dm_buf, offset, len, write,
1144 __RET_ADDR);
1145 break;
1146 case KASAN_DMA_RAW:
1147 break;
1148 default:
1149 panic("%s: impossible", __func__);
1150 }
1151}
1152
1153 void
1154 kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type)
1155{
1156 map->dm_buf = buf;
1157 map->dm_buflen = buflen;
1158 map->dm_buftype = type;
1159}
1160
1161 /* -------------------------------------------------------------------------- */
1162
1163 void __asan_register_globals(struct __asan_global *, size_t);
1164 void __asan_unregister_globals(struct __asan_global *, size_t);
1165
1166 void
1167 __asan_register_globals(struct __asan_global *globals, size_t n)
1168{
1169 size_t i;
1170
1171 for (i = 0; i < n; i++) {
1172 kasan_mark(globals[i].beg, globals[i].size,
1173 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE);
1174 }
1175}
1176
1177 void
1178 __asan_unregister_globals(struct __asan_global *globals, size_t n)
1179{
1180 /* never called */
1181}
1182
1183#define ASAN_LOAD_STORE(size) \
1184 void __asan_load##size(unsigned long); \
1185 void __asan_load##size(unsigned long addr) \
1186 { \
1187 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1188 } \
1189 void __asan_load##size##_noabort(unsigned long); \
1190 void __asan_load##size##_noabort(unsigned long addr) \
1191 { \
1192 kasan_shadow_check(addr, size, false, __RET_ADDR);\
1193 } \
1194 void __asan_store##size(unsigned long); \
1195 void __asan_store##size(unsigned long addr) \
1196 { \
1197 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1198 } \
1199 void __asan_store##size##_noabort(unsigned long); \
1200 void __asan_store##size##_noabort(unsigned long addr) \
1201 { \
1202 kasan_shadow_check(addr, size, true, __RET_ADDR);\
1203 }
1204
1205 ASAN_LOAD_STORE(1);
1206 ASAN_LOAD_STORE(2);
1207 ASAN_LOAD_STORE(4);
1208 ASAN_LOAD_STORE(8);
1209 ASAN_LOAD_STORE(16);
1210
1211 void __asan_loadN(unsigned long, size_t);
1212 void __asan_loadN_noabort(unsigned long, size_t);
1213 void __asan_storeN(unsigned long, size_t);
1214 void __asan_storeN_noabort(unsigned long, size_t);
1215 void __asan_handle_no_return(void);
1216
1217 void
1218 __asan_loadN(unsigned long addr, size_t size)
1219{
1220 kasan_shadow_check(addr, size, false, __RET_ADDR);
1221}
1222
1223 void
1224 __asan_loadN_noabort(unsigned long addr, size_t size)
1225{
1226 kasan_shadow_check(addr, size, false, __RET_ADDR);
1227}
1228
1229 void
1230 __asan_storeN(unsigned long addr, size_t size)
1231{
1232 kasan_shadow_check(addr, size, true, __RET_ADDR);
1233}
1234
1235 void
1236 __asan_storeN_noabort(unsigned long addr, size_t size)
1237{
1238 kasan_shadow_check(addr, size, true, __RET_ADDR);
1239}
1240
1241 void
1242 __asan_handle_no_return(void)
1243{
1244 /* nothing */
1245}
1246
1247#define ASAN_SET_SHADOW(byte) \
1248 void __asan_set_shadow_##byte(void *, size_t); \
1249 void __asan_set_shadow_##byte(void *addr, size_t size) \
1250 { \
1251 __builtin_memset((void *)addr, 0x##byte, size); \
1252 }
1253
1254 ASAN_SET_SHADOW(00);
1255 ASAN_SET_SHADOW(f1);
1256 ASAN_SET_SHADOW(f2);
1257 ASAN_SET_SHADOW(f3);
1258 ASAN_SET_SHADOW(f5);
1259 ASAN_SET_SHADOW(f8);
1260
1261 void __asan_poison_stack_memory(const void *, size_t);
1262 void __asan_unpoison_stack_memory(const void *, size_t);
1263
1264 void
1265 __asan_poison_stack_memory(const void *addr, size_t size)
1266{
1267 size = roundup(size, KASAN_SHADOW_SCALE_SIZE);
1268 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE);
1269}
1270
1271 void
1272 __asan_unpoison_stack_memory(const void *addr, size_t size)
1273{
1274 kasan_shadow_Nbyte_markvalid(addr, size);
1275}
1276
1277 void __asan_alloca_poison(const void *, size_t);
1278 void __asan_allocas_unpoison(const void *, const void *);
1279
1280 void __asan_alloca_poison(const void *addr, size_t size)
1281{
1282 const void *l, *r;
1283
1284 KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0);
1285
1286 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE;
1287 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE);
1288
1289 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT);
1290 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE),
1291 KASAN_STACK_MID);
1292 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT);
1293}
1294
1295 void __asan_allocas_unpoison(const void *stkbegin, const void *stkend)
1296{
1297 size_t size;
1298
1299 if (__predict_false(!stkbegin))
1300 return;
1301 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend))
1302 return;
1303 size = (uintptr_t)stkend - (uintptr_t)stkbegin;
1304
1305 kasan_shadow_Nbyte_fill(stkbegin, size, 0);
1306}
1307
Super User's BSD Cross Reference: /NetBSD/sys/kern/subr_asan.c
