-
Notifications
You must be signed in to change notification settings - Fork 1.5k
/
Copy pathxmemory
2665 lines (2264 loc) · 103 KB
/
xmemory
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// xmemory internal header
// Copyright (c) Microsoft Corporation.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#ifndef _XMEMORY_
#define _XMEMORY_
#include <yvals_core.h>
#if _STL_COMPILER_PREPROCESSOR
#include <cstdint>
#include <cstdlib>
#include <limits> // TRANSITION, see GH-4634: Lots of user code assumes that <xmemory> drags in <limits>
#include <new>
#include <xatomic.h>
#include <xutility>
#if _HAS_CXX20
#include <tuple>
#endif // _HAS_CXX20
#pragma pack(push, _CRT_PACKING)
#pragma warning(push, _STL_WARNING_LEVEL)
#pragma warning(disable : _STL_DISABLED_WARNINGS)
_STL_DISABLE_CLANG_WARNINGS
#pragma push_macro("new")
#undef new
_STD_BEGIN
template <class _Ptrty>
_NODISCARD constexpr auto _Unfancy(_Ptrty _Ptr) noexcept { // converts from a fancy pointer to a plain pointer
return _STD addressof(*_Ptr);
}
template <class _Ty>
_NODISCARD constexpr _Ty* _Unfancy(_Ty* _Ptr) noexcept { // do nothing for plain pointers
return _Ptr;
}
template <class _Ptrty>
constexpr auto _Unfancy_maybe_null(_Ptrty _Ptr) noexcept {
// converts from a (potentially null) fancy pointer to a plain pointer
return _Ptr ? _STD addressof(*_Ptr) : nullptr;
}
template <class _Ty>
constexpr _Ty* _Unfancy_maybe_null(_Ty* _Ptr) noexcept { // do nothing for plain pointers
return _Ptr;
}
template <class _Ty>
struct _NODISCARD _Tidy_guard { // class with destructor that calls _Tidy
_Ty* _Target;
_CONSTEXPR20 ~_Tidy_guard() {
if (_Target) {
_Target->_Tidy();
}
}
};
template <class _Ty>
struct _NODISCARD _Tidy_deallocate_guard { // class with destructor that calls _Tidy_deallocate
_Ty* _Target;
_CONSTEXPR20 ~_Tidy_deallocate_guard() {
if (_Target) {
_Target->_Tidy_deallocate();
}
}
};
template <class _Keycmp, class _Lhs, class _Rhs>
constexpr bool _Nothrow_compare = noexcept(
static_cast<bool>(_STD declval<const _Keycmp&>()(_STD declval<const _Lhs&>(), _STD declval<const _Rhs&>())));
[[noreturn]] inline void _Throw_bad_array_new_length() {
_THROW(bad_array_new_length{});
}
template <size_t _Ty_size>
_NODISCARD constexpr size_t _Get_size_of_n(const size_t _Count) {
constexpr bool _Overflow_is_possible = _Ty_size > 1;
if constexpr (_Overflow_is_possible) {
constexpr size_t _Max_possible = static_cast<size_t>(-1) / _Ty_size;
if (_Count > _Max_possible) {
_Throw_bad_array_new_length(); // multiply overflow
}
}
return _Count * _Ty_size;
}
template <class _Ty>
constexpr size_t _New_alignof = (_STD max)(alignof(_Ty), __STDCPP_DEFAULT_NEW_ALIGNMENT__);
struct _Default_allocate_traits {
__declspec(allocator) static
#ifdef __clang__ // Clang and MSVC implement P0784R7 differently; see GH-1532
_CONSTEXPR20
#endif // defined(__clang__)
void*
_Allocate(const size_t _Bytes) {
return ::operator new(_Bytes);
}
#ifdef __cpp_aligned_new
__declspec(allocator) static
#ifdef __clang__ // Clang and MSVC implement P0784R7 differently; see GH-1532
_CONSTEXPR20
#endif // defined(__clang__)
void*
_Allocate_aligned(const size_t _Bytes, const size_t _Align) {
#ifdef __clang__ // Clang and MSVC implement P0784R7 differently; see GH-1532
#if _HAS_CXX20
if (_STD is_constant_evaluated()) {
return ::operator new(_Bytes);
} else
#endif // _HAS_CXX20
#endif // defined(__clang__)
{
return ::operator new(_Bytes, align_val_t{_Align});
}
}
#endif // defined(__cpp_aligned_new)
};
constexpr bool _Is_pow_2(const size_t _Value) noexcept {
return _Value != 0 && (_Value & (_Value - 1)) == 0;
}
#if defined(_M_IX86) || defined(_M_X64)
_INLINE_VAR constexpr size_t _Big_allocation_threshold = 4096;
_INLINE_VAR constexpr size_t _Big_allocation_alignment = 32;
// Big allocation alignment should at least match vector register alignment
_STL_INTERNAL_STATIC_ASSERT(2 * sizeof(void*) <= _Big_allocation_alignment);
// Big allocation alignment must be a power of two
_STL_INTERNAL_STATIC_ASSERT(_Is_pow_2(_Big_allocation_alignment));
#ifdef _DEBUG
_INLINE_VAR constexpr size_t _Non_user_size = 2 * sizeof(void*) + _Big_allocation_alignment - 1;
#else // ^^^ defined(_DEBUG) / !defined(_DEBUG) vvv
_INLINE_VAR constexpr size_t _Non_user_size = sizeof(void*) + _Big_allocation_alignment - 1;
#endif // ^^^ !defined(_DEBUG) ^^^
#ifdef _WIN64
_INLINE_VAR constexpr size_t _Big_allocation_sentinel = 0xFAFAFAFAFAFAFAFAULL;
#else // ^^^ defined(_WIN64) / !defined(_WIN64) vvv
_INLINE_VAR constexpr size_t _Big_allocation_sentinel = 0xFAFAFAFAUL;
#endif // ^^^ !defined(_WIN64) ^^^
template <class _Traits>
__declspec(allocator) void* _Allocate_manually_vector_aligned(const size_t _Bytes) {
// allocate _Bytes manually aligned to at least _Big_allocation_alignment
const size_t _Block_size = _Non_user_size + _Bytes;
if (_Block_size <= _Bytes) {
_Throw_bad_array_new_length(); // add overflow
}
const uintptr_t _Ptr_container = reinterpret_cast<uintptr_t>(_Traits::_Allocate(_Block_size));
_STL_VERIFY(_Ptr_container != 0, "invalid argument"); // validate even in release since we're doing p[-1]
void* const _Ptr = reinterpret_cast<void*>((_Ptr_container + _Non_user_size) & ~(_Big_allocation_alignment - 1));
static_cast<uintptr_t*>(_Ptr)[-1] = _Ptr_container;
#ifdef _DEBUG
static_cast<uintptr_t*>(_Ptr)[-2] = _Big_allocation_sentinel;
#endif // defined(_DEBUG)
return _Ptr;
}
inline void _Adjust_manually_vector_aligned(void*& _Ptr, size_t& _Bytes) {
// adjust parameters from _Allocate_manually_vector_aligned to pass to operator delete
_Bytes += _Non_user_size;
const uintptr_t* const _Ptr_user = static_cast<uintptr_t*>(_Ptr);
const uintptr_t _Ptr_container = _Ptr_user[-1];
// If the following asserts, it likely means that we are performing
// an aligned delete on memory coming from an unaligned allocation.
_STL_ASSERT(_Ptr_user[-2] == _Big_allocation_sentinel, "invalid argument");
// Extra paranoia on aligned allocation/deallocation; ensure _Ptr_container is
// in range [_Min_back_shift, _Non_user_size]
#ifdef _DEBUG
constexpr uintptr_t _Min_back_shift = 2 * sizeof(void*);
#else // ^^^ defined(_DEBUG) / !defined(_DEBUG) vvv
constexpr uintptr_t _Min_back_shift = sizeof(void*);
#endif // ^^^ !defined(_DEBUG) ^^^
const uintptr_t _Back_shift = reinterpret_cast<uintptr_t>(_Ptr) - _Ptr_container;
_STL_VERIFY(_Back_shift >= _Min_back_shift && _Back_shift <= _Non_user_size, "invalid argument");
_Ptr = reinterpret_cast<void*>(_Ptr_container);
}
#endif // defined(_M_IX86) || defined(_M_X64)
template <size_t _Align, class _Traits = _Default_allocate_traits>
__declspec(allocator) _CONSTEXPR20 void* _Allocate(const size_t _Bytes) {
// allocate _Bytes
if (_Bytes == 0) {
return nullptr;
}
#if _HAS_CXX20 // TRANSITION, GH-1532
if (_STD is_constant_evaluated()) {
return _Traits::_Allocate(_Bytes);
}
#endif // _HAS_CXX20
#ifdef __cpp_aligned_new
if constexpr (_Align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) {
size_t _Passed_align = _Align;
#if defined(_M_IX86) || defined(_M_X64)
if (_Bytes >= _Big_allocation_threshold) {
// boost the alignment of big allocations to help autovectorization
_Passed_align = (_STD max)(_Align, _Big_allocation_alignment);
}
#endif // defined(_M_IX86) || defined(_M_X64)
return _Traits::_Allocate_aligned(_Bytes, _Passed_align);
} else
#endif // defined(__cpp_aligned_new)
{
#if defined(_M_IX86) || defined(_M_X64)
if (_Bytes >= _Big_allocation_threshold) {
// boost the alignment of big allocations to help autovectorization
return _Allocate_manually_vector_aligned<_Traits>(_Bytes);
}
#endif // defined(_M_IX86) || defined(_M_X64)
return _Traits::_Allocate(_Bytes);
}
}
template <size_t _Align>
_CONSTEXPR20 void _Deallocate(void* _Ptr, size_t _Bytes) noexcept {
// deallocate storage allocated by _Allocate
#if _HAS_CXX20 // TRANSITION, GH-1532
if (_STD is_constant_evaluated()) {
::operator delete(_Ptr);
return;
}
#endif // _HAS_CXX20
#ifdef __cpp_aligned_new
if constexpr (_Align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) {
size_t _Passed_align = _Align;
#if defined(_M_IX86) || defined(_M_X64)
if (_Bytes >= _Big_allocation_threshold) {
// boost the alignment of big allocations to help autovectorization
_Passed_align = (_STD max)(_Align, _Big_allocation_alignment);
}
#endif // defined(_M_IX86) || defined(_M_X64)
::operator delete(_Ptr, _Bytes, align_val_t{_Passed_align});
} else
#endif // defined(__cpp_aligned_new)
{
#if defined(_M_IX86) || defined(_M_X64)
if (_Bytes >= _Big_allocation_threshold) {
// boost the alignment of big allocations to help autovectorization
_Adjust_manually_vector_aligned(_Ptr, _Bytes);
}
#endif // defined(_M_IX86) || defined(_M_X64)
::operator delete(_Ptr, _Bytes);
}
}
template <class _Ptr, class _Ty>
using _Rebind_pointer_t = typename pointer_traits<_Ptr>::template rebind<_Ty>;
template <class _Pointer, enable_if_t<!is_pointer_v<_Pointer>, int> = 0>
_CONSTEXPR20 _Pointer _Refancy(typename pointer_traits<_Pointer>::element_type* _Ptr) noexcept {
return pointer_traits<_Pointer>::pointer_to(*_Ptr);
}
template <class _Pointer, enable_if_t<is_pointer_v<_Pointer>, int> = 0>
_CONSTEXPR20 _Pointer _Refancy(_Pointer _Ptr) noexcept {
return _Ptr;
}
template <class _Pointer, enable_if_t<!is_pointer_v<_Pointer>, int> = 0>
_CONSTEXPR20 _Pointer _Refancy_maybe_null(typename pointer_traits<_Pointer>::element_type* _Ptr) noexcept {
return _Ptr == nullptr ? _Pointer() : pointer_traits<_Pointer>::pointer_to(*_Ptr);
}
template <class _Pointer, enable_if_t<is_pointer_v<_Pointer>, int> = 0>
_CONSTEXPR20 _Pointer _Refancy_maybe_null(_Pointer _Ptr) noexcept {
return _Ptr;
}
template <class _NoThrowFwdIt, class _NoThrowSentinel>
_CONSTEXPR20 void _Destroy_range(_NoThrowFwdIt _First, _NoThrowSentinel _Last) noexcept;
template <class _Ty>
_CONSTEXPR20 void _Destroy_in_place(_Ty& _Obj) noexcept {
if constexpr (is_array_v<_Ty>) {
_STD _Destroy_range(_Obj, _Obj + extent_v<_Ty>);
} else {
_Obj.~_Ty();
}
}
#if _HAS_CXX17
_EXPORT_STD template <class _Ty>
_CONSTEXPR20 void destroy_at(_Ty* const _Location) noexcept /* strengthened */ {
#if _HAS_CXX20
if constexpr (is_array_v<_Ty>) {
_STD _Destroy_range(_STD begin(*_Location), _STD end(*_Location));
} else
#endif // _HAS_CXX20
{
_Location->~_Ty();
}
}
#endif // _HAS_CXX17
template <class _Ptrty>
auto _Const_cast(_Ptrty _Ptr) noexcept { // remove constness from a fancy pointer
using _Elem = typename pointer_traits<_Ptrty>::element_type;
using _Modifiable = remove_const_t<_Elem>;
using _Dest = typename pointer_traits<_Ptrty>::template rebind<_Modifiable>;
return pointer_traits<_Dest>::pointer_to(const_cast<_Modifiable&>(*_Ptr));
}
template <class _Ty>
auto _Const_cast(_Ty* _Ptr) noexcept {
return const_cast<remove_const_t<_Ty>*>(_Ptr);
}
template <class _Ty, class = void>
struct _Get_pointer_type {
using type = typename _Ty::value_type*;
};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Ty>
struct _Get_pointer_type<_Ty, void_t<typename _Ty::pointer>> {
using type = typename _Ty::pointer;
};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Ty, class = void>
struct _Get_const_pointer_type {
using _Ptrty = typename _Get_pointer_type<_Ty>::type;
using _Valty = typename _Ty::value_type;
using type = typename pointer_traits<_Ptrty>::template rebind<const _Valty>;
};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Ty>
struct _Get_const_pointer_type<_Ty, void_t<typename _Ty::const_pointer>> {
using type = typename _Ty::const_pointer;
};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Ty, class = void>
struct _Get_void_pointer_type {
using _Ptrty = typename _Get_pointer_type<_Ty>::type;
using type = typename pointer_traits<_Ptrty>::template rebind<void>;
};
template <class _Ty>
struct _Get_void_pointer_type<_Ty, void_t<typename _Ty::void_pointer>> {
using type = typename _Ty::void_pointer;
};
template <class _Ty, class = void>
struct _Get_const_void_pointer_type {
using _Ptrty = typename _Get_pointer_type<_Ty>::type;
using type = typename pointer_traits<_Ptrty>::template rebind<const void>;
};
template <class _Ty>
struct _Get_const_void_pointer_type<_Ty, void_t<typename _Ty::const_void_pointer>> {
using type = typename _Ty::const_void_pointer;
};
template <class _Ty, class = void>
struct _Get_difference_type {
using _Ptrty = typename _Get_pointer_type<_Ty>::type;
using type = typename pointer_traits<_Ptrty>::difference_type;
};
template <class _Ty>
struct _Get_difference_type<_Ty, void_t<typename _Ty::difference_type>> {
using type = typename _Ty::difference_type;
};
template <class _Ty, class = void>
struct _Get_size_type {
using type = make_unsigned_t<typename _Get_difference_type<_Ty>::type>;
};
template <class _Ty>
struct _Get_size_type<_Ty, void_t<typename _Ty::size_type>> {
using type = typename _Ty::size_type;
};
template <class _Ty, class = void>
struct _Get_propagate_on_container_copy {
using type = false_type;
};
template <class _Ty>
struct _Get_propagate_on_container_copy<_Ty, void_t<typename _Ty::propagate_on_container_copy_assignment>> {
using type = typename _Ty::propagate_on_container_copy_assignment;
};
template <class _Ty, class = void>
struct _Get_propagate_on_container_move {
using type = false_type;
};
template <class _Ty>
struct _Get_propagate_on_container_move<_Ty, void_t<typename _Ty::propagate_on_container_move_assignment>> {
using type = typename _Ty::propagate_on_container_move_assignment;
};
template <class _Ty, class = void>
struct _Get_propagate_on_container_swap {
using type = false_type;
};
template <class _Ty>
struct _Get_propagate_on_container_swap<_Ty, void_t<typename _Ty::propagate_on_container_swap>> {
using type = typename _Ty::propagate_on_container_swap;
};
template <class _Ty, class = void>
struct _Get_is_always_equal {
using type = bool_constant<is_empty_v<_Ty>>;
};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Ty>
struct _Get_is_always_equal<_Ty, void_t<typename _Ty::is_always_equal>> {
using type = typename _Ty::is_always_equal;
};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Ty, class _Other, class = void>
struct _Get_rebind_type {
using type = typename _Replace_first_parameter<_Other, _Ty>::type;
};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Ty, class _Other>
struct _Get_rebind_type<_Ty, _Other, void_t<typename _Ty::template rebind<_Other>::other>> {
using type = typename _Ty::template rebind<_Other>::other;
};
_STL_RESTORE_DEPRECATED_WARNING
_EXPORT_STD template <class _Ty>
class allocator;
template <class _Alloc, class = void>
struct _Is_default_allocator : false_type {};
template <class _Ty>
struct _Is_default_allocator<allocator<_Ty>, void_t<typename allocator<_Ty>::_From_primary>>
: is_same<typename allocator<_Ty>::_From_primary, allocator<_Ty>>::type {};
#if _HAS_CXX23
template <class _Alloc, class _SizeTy>
concept _Has_member_allocate_at_least = requires(_Alloc& _Al, const _SizeTy& _Count) { _Al.allocate_at_least(_Count); };
#endif // _HAS_CXX23
template <class _Void, class... _Types>
struct _Has_no_allocator_construct : true_type {};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Alloc, class _Ptr, class... _Args>
struct _Has_no_allocator_construct<
void_t<decltype(_STD declval<_Alloc&>().construct(_STD declval<_Ptr>(), _STD declval<_Args>()...))>, _Alloc, _Ptr,
_Args...> : false_type {};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Alloc, class _Ptr, class... _Args>
using _Uses_default_construct =
disjunction<_Is_default_allocator<_Alloc>, _Has_no_allocator_construct<void, _Alloc, _Ptr, _Args...>>;
template <class _Alloc, class _Ptr, class = void>
struct _Has_no_alloc_destroy : true_type {};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Alloc, class _Ptr>
struct _Has_no_alloc_destroy<_Alloc, _Ptr, void_t<decltype(_STD declval<_Alloc&>().destroy(_STD declval<_Ptr>()))>>
: false_type {};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Alloc, class _Ptr>
using _Uses_default_destroy = disjunction<_Is_default_allocator<_Alloc>, _Has_no_alloc_destroy<_Alloc, _Ptr>>;
template <class _Alloc, class _Size_type, class _Const_void_pointer, class = void>
struct _Has_allocate_hint : false_type {};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Alloc, class _Size_type, class _Const_void_pointer>
struct _Has_allocate_hint<_Alloc, _Size_type, _Const_void_pointer,
void_t<decltype(_STD declval<_Alloc&>().allocate(
_STD declval<const _Size_type&>(), _STD declval<const _Const_void_pointer&>()))>> : true_type {};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Alloc, class = void>
struct _Has_max_size : false_type {};
_STL_DISABLE_DEPRECATED_WARNING
template <class _Alloc>
struct _Has_max_size<_Alloc, void_t<decltype(_STD declval<const _Alloc&>().max_size())>> : true_type {};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Alloc, class = void>
struct _Has_select_on_container_copy_construction : false_type {};
template <class _Alloc>
struct _Has_select_on_container_copy_construction<_Alloc,
void_t<decltype(_STD declval<const _Alloc&>().select_on_container_copy_construction())>> : true_type {};
#if _HAS_CXX23
_EXPORT_STD template <class _Ptr, class _SizeTy = size_t>
struct allocation_result {
_Ptr ptr;
_SizeTy count;
};
#endif // _HAS_CXX23
_EXPORT_STD template <class _Alloc>
struct allocator_traits;
_STL_DISABLE_DEPRECATED_WARNING
template <class _Alloc>
struct _Normal_allocator_traits { // defines traits for allocators
using allocator_type = _Alloc;
using value_type = typename _Alloc::value_type;
using pointer = typename _Get_pointer_type<_Alloc>::type;
using const_pointer = typename _Get_const_pointer_type<_Alloc>::type;
using void_pointer = typename _Get_void_pointer_type<_Alloc>::type;
using const_void_pointer = typename _Get_const_void_pointer_type<_Alloc>::type;
using size_type = typename _Get_size_type<_Alloc>::type;
using difference_type = typename _Get_difference_type<_Alloc>::type;
using propagate_on_container_copy_assignment = typename _Get_propagate_on_container_copy<_Alloc>::type;
using propagate_on_container_move_assignment = typename _Get_propagate_on_container_move<_Alloc>::type;
using propagate_on_container_swap = typename _Get_propagate_on_container_swap<_Alloc>::type;
using is_always_equal = typename _Get_is_always_equal<_Alloc>::type;
template <class _Other>
using rebind_alloc = typename _Get_rebind_type<_Alloc, _Other>::type;
template <class _Other>
using rebind_traits = allocator_traits<rebind_alloc<_Other>>;
_NODISCARD_RAW_PTR_ALLOC static _CONSTEXPR20 __declspec(allocator) pointer allocate(
_Alloc& _Al, _CRT_GUARDOVERFLOW const size_type _Count) {
return _Al.allocate(_Count);
}
_NODISCARD_RAW_PTR_ALLOC static _CONSTEXPR20 __declspec(allocator) pointer allocate(
_Alloc& _Al, _CRT_GUARDOVERFLOW const size_type _Count, const const_void_pointer _Hint) {
if constexpr (_Has_allocate_hint<_Alloc, size_type, const_void_pointer>::value) {
return _Al.allocate(_Count, _Hint);
} else {
return _Al.allocate(_Count);
}
}
#if _HAS_CXX23
_NODISCARD_RAW_PTR_ALLOC static constexpr allocation_result<pointer, size_type> allocate_at_least(
_Alloc& _Al, _CRT_GUARDOVERFLOW const size_type _Count) {
if constexpr (_Has_member_allocate_at_least<_Alloc, size_type>) {
return _Al.allocate_at_least(_Count);
} else {
return {_Al.allocate(_Count), _Count};
}
}
#endif // _HAS_CXX23
static _CONSTEXPR20 void deallocate(_Alloc& _Al, pointer _Ptr, size_type _Count) {
_Al.deallocate(_Ptr, _Count);
}
template <class _Ty, class... _Types>
static _CONSTEXPR20 void construct(_Alloc& _Al, _Ty* _Ptr, _Types&&... _Args) {
if constexpr (_Uses_default_construct<_Alloc, _Ty*, _Types...>::value) {
#if _HAS_CXX20
_STD construct_at(_Ptr, _STD forward<_Types>(_Args)...);
#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv
::new (static_cast<void*>(_Ptr)) _Ty(_STD forward<_Types>(_Args)...);
#endif // ^^^ !_HAS_CXX20 ^^^
} else {
_Al.construct(_Ptr, _STD forward<_Types>(_Args)...);
}
}
template <class _Ty>
static _CONSTEXPR20 void destroy(_Alloc& _Al, _Ty* _Ptr) {
if constexpr (_Uses_default_destroy<_Alloc, _Ty*>::value) {
#if _HAS_CXX20
_STD destroy_at(_Ptr);
#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv
_Ptr->~_Ty();
#endif // ^^^ !_HAS_CXX20 ^^^
} else {
_Al.destroy(_Ptr);
}
}
_NODISCARD static _CONSTEXPR20 size_type max_size(const _Alloc& _Al) noexcept {
if constexpr (_Has_max_size<_Alloc>::value) {
return _Al.max_size();
} else {
return _STD _Max_limit<size_type>() / sizeof(value_type);
}
}
_NODISCARD static _CONSTEXPR20 _Alloc select_on_container_copy_construction(const _Alloc& _Al) {
if constexpr (_Has_select_on_container_copy_construction<_Alloc>::value) {
return _Al.select_on_container_copy_construction();
} else {
return _Al;
}
}
};
_STL_RESTORE_DEPRECATED_WARNING
template <class _Alloc>
struct _Default_allocator_traits { // traits for std::allocator
using allocator_type = _Alloc;
using value_type = typename _Alloc::value_type;
using pointer = value_type*;
using const_pointer = const value_type*;
using void_pointer = void*;
using const_void_pointer = const void*;
using size_type = size_t;
using difference_type = ptrdiff_t;
using propagate_on_container_copy_assignment = false_type;
using propagate_on_container_move_assignment = true_type;
using propagate_on_container_swap = false_type;
using is_always_equal = true_type;
template <class _Other>
using rebind_alloc = allocator<_Other>;
template <class _Other>
using rebind_traits = allocator_traits<allocator<_Other>>;
_NODISCARD_RAW_PTR_ALLOC static _CONSTEXPR20 __declspec(allocator) pointer allocate(
_Alloc& _Al, _CRT_GUARDOVERFLOW const size_type _Count) {
#if _HAS_CXX20 // TRANSITION, GH-1532
if (_STD is_constant_evaluated()) {
return _Al.allocate(_Count);
} else
#endif // _HAS_CXX20
{
(void) _Al;
return static_cast<pointer>(
_Allocate<_New_alignof<value_type>>(_Get_size_of_n<sizeof(value_type)>(_Count)));
}
}
_NODISCARD_RAW_PTR_ALLOC static _CONSTEXPR20 __declspec(allocator) pointer allocate(
_Alloc& _Al, _CRT_GUARDOVERFLOW const size_type _Count, const_void_pointer) {
#if _HAS_CXX20 // TRANSITION, GH-1532
if (_STD is_constant_evaluated()) {
return _Al.allocate(_Count);
} else
#endif // _HAS_CXX20
{
(void) _Al;
return static_cast<pointer>(
_Allocate<_New_alignof<value_type>>(_Get_size_of_n<sizeof(value_type)>(_Count)));
}
}
#if _HAS_CXX23
_NODISCARD_RAW_PTR_ALLOC static constexpr allocation_result<pointer, size_type> allocate_at_least(
_Alloc& _Al, _CRT_GUARDOVERFLOW const size_type _Count) {
return {_Al.allocate(_Count), _Count};
}
#endif // _HAS_CXX23
static _CONSTEXPR20 void deallocate(_Alloc& _Al, const pointer _Ptr, const size_type _Count) {
// no overflow check on the following multiply; we assume _Allocate did that check
#if _HAS_CXX20 // TRANSITION, GH-1532
if (_STD is_constant_evaluated()) {
_Al.deallocate(_Ptr, _Count);
} else
#endif // _HAS_CXX20
{
(void) _Al;
_STD _Deallocate<_New_alignof<value_type>>(_Ptr, sizeof(value_type) * _Count);
}
}
template <class _Objty, class... _Types>
static _CONSTEXPR20 void construct(_Alloc&, _Objty* const _Ptr, _Types&&... _Args) {
#if _HAS_CXX20
_STD construct_at(_Ptr, _STD forward<_Types>(_Args)...);
#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv
::new (const_cast<void*>(static_cast<const volatile void*>(_Ptr))) _Objty(_STD forward<_Types>(_Args)...);
#endif // ^^^ !_HAS_CXX20 ^^^
}
template <class _Uty>
static _CONSTEXPR20 void destroy(_Alloc&, _Uty* const _Ptr) {
#if _HAS_CXX20
_STD destroy_at(_Ptr);
#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv
_Ptr->~_Uty();
#endif // ^^^ !_HAS_CXX20 ^^^
}
_NODISCARD static _CONSTEXPR20 size_type max_size(const _Alloc&) noexcept {
return static_cast<size_t>(-1) / sizeof(value_type);
}
_NODISCARD static _CONSTEXPR20 _Alloc select_on_container_copy_construction(const _Alloc& _Al) {
return _Al;
}
};
_EXPORT_STD template <class _Alloc>
struct allocator_traits : conditional_t<_Is_default_allocator<_Alloc>::value, _Default_allocator_traits<_Alloc>,
_Normal_allocator_traits<_Alloc>> {};
// _Choose_pocca_v returns whether an attempt to propagate allocators is necessary in copy assignment operations.
// Note that even when false_type, callers should call _Pocca as we want to assign allocators even when equal.
template <class _Alloc>
constexpr bool _Choose_pocca_v = allocator_traits<_Alloc>::propagate_on_container_copy_assignment::value
&& !allocator_traits<_Alloc>::is_always_equal::value;
enum class _Pocma_values {
_Equal_allocators, // usually allows contents to be stolen (e.g. with swap)
_Propagate_allocators, // usually allows the allocator to be propagated, and then contents stolen
_No_propagate_allocators, // usually turns moves into copies
};
template <class _Alloc>
constexpr _Pocma_values _Choose_pocma_v = allocator_traits<_Alloc>::is_always_equal::value
? _Pocma_values::_Equal_allocators
: (allocator_traits<_Alloc>::propagate_on_container_move_assignment::value
? _Pocma_values::_Propagate_allocators
: _Pocma_values::_No_propagate_allocators);
template <class _Alloc, class _Value_type>
using _Rebind_alloc_t = typename allocator_traits<_Alloc>::template rebind_alloc<_Value_type>;
// If _Alloc is already rebound appropriately, binds an lvalue reference to it, avoiding a copy. Otherwise, creates a
// rebound copy.
template <class _Alloc, class _Value_type>
using _Maybe_rebind_alloc_t =
typename _Select<is_same_v<typename _Alloc::value_type, _Value_type>>::template _Apply<_Alloc&,
_Rebind_alloc_t<_Alloc, _Value_type>>;
template <class _Alloc> // tests if allocator has simple addressing
constexpr bool _Is_simple_alloc_v =
is_same_v<typename allocator_traits<_Alloc>::size_type, size_t>
&& is_same_v<typename allocator_traits<_Alloc>::difference_type, ptrdiff_t>
&& is_same_v<typename allocator_traits<_Alloc>::pointer, typename _Alloc::value_type*>
&& is_same_v<typename allocator_traits<_Alloc>::const_pointer, const typename _Alloc::value_type*>;
template <class _Value_type>
struct _Simple_types { // wraps types from allocators with simple addressing for use in iterators
// and other SCARY machinery
using value_type = _Value_type;
using size_type = size_t;
using difference_type = ptrdiff_t;
using pointer = value_type*;
using const_pointer = const value_type*;
};
// The number of user bytes a single byte of ASAN shadow memory can track.
_INLINE_VAR constexpr size_t _Asan_granularity = 8;
_INLINE_VAR constexpr size_t _Asan_granularity_mask = _Asan_granularity - 1;
struct _Asan_aligned_pointers {
const void* _First;
const void* _End;
_NODISCARD constexpr const void* _Clamp_to_end(const void* _Mid) const noexcept {
_STL_INTERNAL_CHECK(_Mid >= _First);
if (_Mid > _End) {
return _End;
} else {
return _Mid;
}
}
};
// The way that ASan shadow memory works, each eight byte block of memory ("shadow memory section")
// has a single byte to mark it as either poison or valid.
// Each section has 0 to 8 "valid" bytes followed by poison bytes, so:
// ```
// [ v v v p p p p p ]
// ```
// or
// ```
// [ v v v v v v v v ]
// ```
// are okay, but
// ```
// [ p p p p v v v v ]
// ```
// is not.
//
// This function exists to fix up `first` and `end` pointers so that one can call
// `__sanitizer_annotate_contiguous_container`:
//
// - `__sanitizer_annotate_contiguous_container` checks that `first` is aligned to an 8-byte boundary
// - if `end` is not aligned to an 8-byte boundary, `__sanitizer_annotate_contiguous_container` still poisons the
// remaining bytes in the shadow memory section.
//
// Because of the second property, we can only mark poison up to the final aligned address before the true `last`.
// Otherwise, we'd poison the memory _after_ `last` as well.
// For the first property, we can assume that everything before `first` in the shadow memory section is valid
// (since otherwise we couldn't mark `first` valid), and so we just return back the first address in
// `first`'s shadow memory section.
//
// ### Example
//
// ```cpp
// struct alignas(8) cat {
// int meow; // bytes [0, 4)
// char buffer[16]; // bytes [4, 20)
// int purr; // bytes [20, 24)
// };
// ```
//
// First, `meow` and `purr` are just regular data members, not container buffers, so they _must_ be valid.
// Then, assume we want to poison all of `buffer`.
// This would mean that, in a perfect world, we want something like:
//
// ```
// | meow | buffer | purr |
// [ v v v v p p p p ][ p p p p p p p p ][ p p p p v v v v ]
// sm1 sm2 sm3
// ```
//
// However, note that by the rules above, `sm3` is not a valid shadow memory section; we always need
// the valid bytes to come before the poison bytes. Thus, the closest we can actually get to it is:
//
// ```
// | meow | buffer | purr |
// [ v v v v p p p p ][ p p p p p p p p ][ v v v v v v v v ]
// sm1 sm2 sm3
// ```
//
// We call `aligned = _Get_asan_aligned_first_end(cat.buffer, cat.buffer + 16);`, and we get back
//
// ```cpp
// aligned = {
// ._First = &cat.meow,
// ._End = cat.buffer + 12,
// };
// ```
//
// Then, we poison as much of buffer as we can via
//
// ```cpp
// __sanitizer_annotate_contiguous_container(
// aligned._First,
// aligned._End,
// cat.buffer,
// aligned._Clamp_to_end(cat.buffer + 16));
// ```
//
// We are allowed to assume that `&cat.meow` is valid, since otherwise `cat.buffer + [0, 4)` could not be valid.
// We cannot poison up to `cat.buffer + 16`, since then `&purr` could not be valid.
// Thus, this results in the shadow memory state from the second example.
_NODISCARD inline _Asan_aligned_pointers _Get_asan_aligned_first_end(
const void* const _First, const void* const _End) noexcept {
return {
reinterpret_cast<const void*>(reinterpret_cast<uintptr_t>(_First) & ~_Asan_granularity_mask),
reinterpret_cast<const void*>(reinterpret_cast<uintptr_t>(_End) & ~_Asan_granularity_mask),
};
}
// When we can assume that the allocator we are using will always align allocations to the 8-byte,
// we can simply push the `_End` pointer to the end of the shadow memory section.
// This is _not_ safe in general (see _Get_asan_aligned_first_end's comment for why).
_NODISCARD inline const void* _Get_asan_aligned_after(const void* const _End) noexcept {
return reinterpret_cast<const void*>(
(reinterpret_cast<uintptr_t>(_End) + _Asan_granularity_mask) & ~_Asan_granularity_mask);
}
template <class _Container, class = void>
constexpr size_t _Container_allocation_minimum_asan_alignment = alignof(typename _Container::value_type);
template <class _Container>
constexpr size_t _Container_allocation_minimum_asan_alignment<_Container,
void_t<decltype(_Container::allocator_type::_Minimum_asan_allocation_alignment)>> =
(_STD max)(
alignof(typename _Container::value_type), _Container::allocator_type::_Minimum_asan_allocation_alignment);
_EXPORT_STD template <class _Ty>
class allocator {
public:
static_assert(!is_const_v<_Ty>, "The C++ Standard forbids containers of const elements "
"because allocator<const T> is ill-formed.");
static_assert(!is_function_v<_Ty>, "The C++ Standard forbids allocators for function elements "
"because of [allocator.requirements].");
static_assert(!is_reference_v<_Ty>, "The C++ Standard forbids allocators for reference elements "
"because of [allocator.requirements].");
using _From_primary = allocator;
using value_type = _Ty;
#if _HAS_DEPRECATED_ALLOCATOR_MEMBERS
using pointer _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS = _Ty*;
using const_pointer _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS = const _Ty*;
using reference _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS = _Ty&;
using const_reference _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS = const _Ty&;
#endif // _HAS_DEPRECATED_ALLOCATOR_MEMBERS
using size_type = size_t;
using difference_type = ptrdiff_t;
using propagate_on_container_move_assignment = true_type;
using is_always_equal _CXX20_DEPRECATE_IS_ALWAYS_EQUAL = true_type;
#if _HAS_DEPRECATED_ALLOCATOR_MEMBERS
template <class _Other>
struct _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS rebind {
using other = allocator<_Other>;
};
_CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS _NODISCARD _Ty* address(_Ty& _Val) const noexcept {
return _STD addressof(_Val);
}
_CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS _NODISCARD const _Ty* address(const _Ty& _Val) const noexcept {
return _STD addressof(_Val);
}
#endif // _HAS_DEPRECATED_ALLOCATOR_MEMBERS
constexpr allocator() noexcept {}
constexpr allocator(const allocator&) noexcept = default;
template <class _Other>
constexpr allocator(const allocator<_Other>&) noexcept {}
_CONSTEXPR20 ~allocator() = default;
_CONSTEXPR20 allocator& operator=(const allocator&) = default;
_CONSTEXPR20 void deallocate(_Ty* const _Ptr, const size_t _Count) {
_STL_ASSERT(_Ptr != nullptr || _Count == 0, "null pointer cannot point to a block of non-zero size");
// no overflow check on the following multiply; we assume _Allocate did that check
_STD _Deallocate<_New_alignof<_Ty>>(_Ptr, sizeof(_Ty) * _Count);
}
_NODISCARD_RAW_PTR_ALLOC _CONSTEXPR20 __declspec(allocator) _Ty* allocate(_CRT_GUARDOVERFLOW const size_t _Count) {
static_assert(sizeof(value_type) > 0, "value_type must be complete before calling allocate.");
return static_cast<_Ty*>(_STD _Allocate<_New_alignof<_Ty>>(_Get_size_of_n<sizeof(_Ty)>(_Count)));
}
#if _HAS_CXX23
_NODISCARD_RAW_PTR_ALLOC constexpr allocation_result<_Ty*> allocate_at_least(
_CRT_GUARDOVERFLOW const size_t _Count) {
return {allocate(_Count), _Count};
}
#endif // _HAS_CXX23
#if _HAS_DEPRECATED_ALLOCATOR_MEMBERS
_CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS _NODISCARD_RAW_PTR_ALLOC __declspec(allocator)
_Ty* allocate(_CRT_GUARDOVERFLOW const size_t _Count, const void*) {
return allocate(_Count);
}
template <class _Objty, class... _Types>
_CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS void construct(_Objty* const _Ptr, _Types&&... _Args) {
::new (const_cast<void*>(static_cast<const volatile void*>(_Ptr))) _Objty(_STD forward<_Types>(_Args)...);
}
template <class _Uty>
_CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS void destroy(_Uty* const _Ptr) {
_Ptr->~_Uty();
}
_CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS _NODISCARD size_t max_size() const noexcept {
return static_cast<size_t>(-1) / sizeof(_Ty);
}
#endif // _HAS_DEPRECATED_ALLOCATOR_MEMBERS
static constexpr size_t _Minimum_asan_allocation_alignment = _Asan_granularity;
};
#if _HAS_DEPRECATED_ALLOCATOR_VOID || _HAS_DEPRECATED_ALLOCATOR_MEMBERS
template <>
class allocator<void> {
public:
using value_type = void;
#if _HAS_DEPRECATED_ALLOCATOR_MEMBERS
using pointer _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS = void*;
using const_pointer _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS = const void*;
template <class _Other>
struct _CXX17_DEPRECATE_OLD_ALLOCATOR_MEMBERS rebind {