reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced

References

   58   typedef uptr Type;

   27   void set(uptr Index, u8 Value) {
   32   u8 operator[](uptr Index) {
   53     for (uptr I = 0; I < Level1Size; I++) {
   68   uptr size() const { return Level1Size * Level2Size; }
   70   void set(uptr Index, u8 Value) {
   77   u8 operator[](uptr Index) const {
   86   u8 *get(uptr Index) const {
   92   u8 *getOrCreate(uptr Index) {

   40 INLINE u16 computeBSDChecksum(u16 Sum, uptr Data) {
   50 WEAK u32 computeHardwareCRC32(u32 Crc, uptr Data);

   23 INLINE u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) {
   23 INLINE u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) {
   23 INLINE u16 computeChecksum(u32 Seed, uptr Value, uptr *Array, uptr ArraySize) {
   36     for (uptr I = 0; I < ArraySize; I++)
   41     for (uptr I = 0; I < ArraySize; I++)
   66   uptr ClassId : 8;
   69   uptr SizeOrUnusedBytes : 20;
   70   uptr Offset : 16;
   71   uptr Checksum : 16;
   78 constexpr uptr ClassIdMask = (1UL << 8) - 1;
   81 constexpr uptr SizeOrUnusedBytesMask = (1UL << 20) - 1;
   82 constexpr uptr OffsetMask = (1UL << 16) - 1;
   83 constexpr uptr ChecksumMask = (1UL << 16) - 1;
   85 constexpr uptr getHeaderSize() {
  107   uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
  107   uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];

   49       const uptr ClassId = NewHeader.ClassId;
   59     void *allocate(UNUSED uptr Size) {
   60       const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
   79       const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(
  160   NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,
  161                           uptr Alignment = MinAlignment,
  178     const uptr NeededSize =
  191     uptr ClassId;
  192     uptr BlockEnd;
  219     uptr UserPtr = reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();
  221       const uptr AlignedUserPtr = roundUpTo(UserPtr, Alignment);
  222       const uptr Offset = AlignedUserPtr - UserPtr;
  247   NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,
  248                            UNUSED uptr Alignment = MinAlignment) {
  280     const uptr Size = getSize(Ptr, &Header);
  289   void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
  289   void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
  315     uptr BlockEnd;
  316     uptr OldSize;
  317     const uptr ClassId = OldHeader.ClassId;
  331       const uptr Delta =
  350       const uptr OldSize = getSize(OldPtr, &OldHeader);
  377   uptr getStats(char *Buffer, uptr Size) {
  377   uptr getStats(char *Buffer, uptr Size) {
  380     const uptr Length = getStats(&Str) + 1;
  404   void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
  404   void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
  407     const uptr From = Base;
  408     const uptr To = Base + Size;
  412       uptr ChunkSize;
  413       const uptr ChunkBase = getChunkFromBlock(Block, &ChunkSize);
  427   bool setOption(UNUSED uptr Option, UNUSED uptr Value) { return false; }
  427   bool setOption(UNUSED uptr Option, UNUSED uptr Value) { return false; }
  434   uptr getUsableSize(const void *Ptr) {
  455   static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;
  456   static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.
  457   static const uptr MinAlignment = 1UL << MinAlignmentLog;
  458   static const uptr MaxAlignment = 1UL << MaxAlignmentLog;
  459   static const uptr MaxAllowedMallocSize =
  464   static const uptr InvalidChunk = ~static_cast<uptr>(0);
  493     const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(
  495     const uptr MaxOffset =
  506     const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;
  511     const uptr LargestClassId = SizeClassMap::LargestClassId;
  525   INLINE uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {
  526     const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;
  538                                    uptr Size) {
  548       const uptr ClassId = NewHeader.ClassId;
  571   uptr getChunkFromBlock(uptr Block, uptr *Size) {
  571   uptr getChunkFromBlock(uptr Block, uptr *Size) {
  571   uptr getChunkFromBlock(uptr Block, uptr *Size) {
  575     const uptr P = Block + Offset + Chunk::getHeaderSize();
  586   uptr getStats(ScopedString *Str) {

   14 uptr PageSizeCached;
   15 uptr getPageSize();
   17 uptr getPageSizeSlow() {

   29 INLINE constexpr uptr roundUpTo(uptr X, uptr Boundary) {
   29 INLINE constexpr uptr roundUpTo(uptr X, uptr Boundary) {
   29 INLINE constexpr uptr roundUpTo(uptr X, uptr Boundary) {
   33 INLINE constexpr uptr roundDownTo(uptr X, uptr Boundary) {
   33 INLINE constexpr uptr roundDownTo(uptr X, uptr Boundary) {
   33 INLINE constexpr uptr roundDownTo(uptr X, uptr Boundary) {
   37 INLINE constexpr bool isAligned(uptr X, uptr Alignment) {
   37 INLINE constexpr bool isAligned(uptr X, uptr Alignment) {
   51 INLINE bool isPowerOfTwo(uptr X) { return (X & (X - 1)) == 0; }
   53 INLINE uptr getMostSignificantSetBitIndex(uptr X) {
   53 INLINE uptr getMostSignificantSetBitIndex(uptr X) {
   58 INLINE uptr roundUpToPowerOfTwo(uptr Size) {
   58 INLINE uptr roundUpToPowerOfTwo(uptr Size) {
   62   const uptr Up = getMostSignificantSetBitIndex(Size);
   68 INLINE uptr getLeastSignificantSetBitIndex(uptr X) {
   68 INLINE uptr getLeastSignificantSetBitIndex(uptr X) {
   73 INLINE uptr getLog2(uptr X) {
   73 INLINE uptr getLog2(uptr X) {
  118 extern uptr PageSizeCached;
  119 uptr getPageSizeSlow();
  120 INLINE uptr getPageSizeCached() {
  137 constexpr uptr MaxRandomLength = 256U;
  138 bool getRandom(void *Buffer, uptr Length, bool Blocking = false);
  154 void *map(void *Addr, uptr Size, const char *Name, uptr Flags = 0,
  154 void *map(void *Addr, uptr Size, const char *Name, uptr Flags = 0,
  161 void unmap(void *Addr, uptr Size, uptr Flags = 0,
  161 void unmap(void *Addr, uptr Size, uptr Flags = 0,
  164 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
  164 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
  164 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,

   14 u32 computeHardwareCRC32(u32 Crc, uptr Data) {

   62   const uptr NameStart = Pos;
   68   const uptr ValueStart = ++Pos;
  101   const uptr OldPos = Pos;
  127     const uptr Len = strlen(Flags[I].Name);

   42   uptr Pos = 0;

   40 uptr getPageSize() { return static_cast<uptr>(sysconf(_SC_PAGESIZE)); }
   44 void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
   44 void *map(void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
   72 void unmap(void *Addr, uptr Size, UNUSED uptr Flags,
   72 void unmap(void *Addr, uptr Size, UNUSED uptr Flags,
   78 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
   78 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
   78 void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
  132 bool getRandom(void *Buffer, uptr Length, UNUSED bool Blocking) {

   36   uptr size() const { return Size; }
   60   uptr Size;
   70     uptr Count = 0;

   41     static u32 getMaxCached(uptr Size) {
   69   void *allocate(uptr ClassId) {
   80     const uptr ClassSize = C->ClassSize;
   90   void deallocate(uptr ClassId, void *P) {
   99     const uptr ClassSize = C->ClassSize;
  106     for (uptr I = 0; I < NumClasses; I++) {
  113   TransferBatch *createBatch(uptr ClassId, void *B) {
  122   static const uptr NumClasses = SizeClassMap::NumClasses;
  126     uptr ClassSize;
  141     for (uptr I = 0; I < NumClasses; I++) {
  143       const uptr Size = SizeClassAllocator::getSizeByClassId(I);
  149   void destroyBatch(uptr ClassId, void *B) {
  154   NOINLINE bool refill(PerClass *C, uptr ClassId) {
  166   NOINLINE void drain(PerClass *C, uptr ClassId) {
  168     const uptr FirstIndexToDrain = C->Count - Count;

   50   static uptr getSizeByClassId(uptr ClassId) {
   50   static uptr getSizeByClassId(uptr ClassId) {
   56   static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
   70     const uptr PageSize = getPageSizeCached();
   71     for (uptr I = 0; I < NumClasses; I++) {
   91     for (uptr I = 0; I < NumRegions; I++)
   97   TransferBatch *popBatch(CacheT *C, uptr ClassId) {
  114   void pushBatch(uptr ClassId, TransferBatch *B) {
  126     for (uptr I = 0; I < NumClasses; I++)
  136     for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++)
  138         const uptr BlockSize = getSizeByClassId(PossibleRegions[I]);
  139         const uptr From = I * RegionSize;
  140         const uptr To = From + (RegionSize / BlockSize) * BlockSize;
  141         for (uptr Block = From; Block < To; Block += BlockSize)
  148     uptr TotalMapped = 0;
  149     uptr PoppedBlocks = 0;
  150     uptr PushedBlocks = 0;
  151     for (uptr I = 0; I < NumClasses; I++) {
  160     for (uptr I = 0; I < NumClasses; I++)
  164   uptr releaseToOS() {
  165     uptr TotalReleasedBytes = 0;
  166     for (uptr I = 0; I < NumClasses; I++) {
  177   static const uptr NumClasses = SizeClassMap::NumClasses;
  178   static const uptr RegionSize = 1UL << RegionSizeLog;
  179   static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> RegionSizeLog;
  187     uptr PoppedBlocks;
  188     uptr PushedBlocks;
  192     uptr PushedBlocksAtLastRelease;
  193     uptr RangesReleased;
  194     uptr LastReleasedBytes;
  204     uptr AllocatedUser;
  209   uptr computeRegionId(uptr Mem) {
  209   uptr computeRegionId(uptr Mem) {
  210     const uptr Id = Mem >> RegionSizeLog;
  215   uptr allocateRegionSlow() {
  216     uptr MapSize = 2 * RegionSize;
  217     const uptr MapBase = reinterpret_cast<uptr>(
  221     const uptr MapEnd = MapBase + MapSize;
  222     uptr Region = MapBase;
  234     const uptr End = Region + MapSize;
  240   uptr allocateRegion(uptr ClassId) {
  240   uptr allocateRegion(uptr ClassId) {
  242     uptr Region = 0;
  252         const uptr RegionIndex = computeRegionId(Region);
  263   SizeClassInfo *getSizeClassInfo(uptr ClassId) {
  268   bool populateBatches(CacheT *C, SizeClassInfo *Sci, uptr ClassId,
  274     for (uptr I = 0; I < Count; I++) {
  291   NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
  293     const uptr Region = allocateRegion(ClassId);
  297     const uptr Size = getSizeByClassId(ClassId);
  300     const uptr NumberOfBlocks = RegionSize / Size;
  303     constexpr uptr ShuffleArraySize = 48;
  306     const uptr AllocatedUser = NumberOfBlocks * Size;
  307     for (uptr I = Region; I < Region + AllocatedUser; I += Size) {
  331   void getStats(ScopedString *Str, uptr ClassId, uptr Rss) {
  331   void getStats(ScopedString *Str, uptr ClassId, uptr Rss) {
  335     const uptr InUse = Sci->Stats.PoppedBlocks - Sci->Stats.PushedBlocks;
  336     const uptr AvailableChunks = Sci->AllocatedUser / getSizeByClassId(ClassId);
  344   NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
  344   NOINLINE uptr releaseToOSMaybe(SizeClassInfo *Sci, uptr ClassId,
  346     const uptr BlockSize = getSizeByClassId(ClassId);
  347     const uptr PageSize = getPageSizeCached();
  350     const uptr BytesInFreeList =
  375     uptr TotalReleasedBytes = 0;
  376     for (uptr I = MinRegionIndex; I <= MaxRegionIndex; I++) {
  398   uptr MinRegionIndex;
  399   uptr MaxRegionIndex;
  403   static constexpr uptr MaxStashedRegions = 4;
  405   uptr NumberOfStashedRegions;
  406   uptr RegionsStash[MaxStashedRegions];

   48   static uptr getSizeByClassId(uptr ClassId) {
   48   static uptr getSizeByClassId(uptr ClassId) {
   54   static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
   69     const uptr PageSize = getPageSizeCached();
   70     for (uptr I = 0; I < NumClasses; I++) {
  100   TransferBatch *popBatch(CacheT *C, uptr ClassId) {
  117   void pushBatch(uptr ClassId, TransferBatch *B) {
  128     for (uptr I = 0; I < NumClasses; I++)
  138     for (uptr I = 0; I < NumClasses; I++) {
  142       const uptr BlockSize = getSizeByClassId(I);
  143       const uptr From = Region->RegionBeg;
  144       const uptr To = From + Region->AllocatedUser;
  145       for (uptr Block = From; Block < To; Block += BlockSize)
  152     uptr TotalMapped = 0;
  153     uptr PoppedBlocks = 0;
  154     uptr PushedBlocks = 0;
  155     for (uptr I = 0; I < NumClasses; I++) {
  167     for (uptr I = 0; I < NumClasses; I++)
  171   uptr releaseToOS() {
  172     uptr TotalReleasedBytes = 0;
  173     for (uptr I = 0; I < NumClasses; I++) {
  184   static const uptr RegionSize = 1UL << RegionSizeLog;
  185   static const uptr NumClasses = SizeClassMap::NumClasses;
  186   static const uptr PrimarySize = RegionSize * NumClasses;
  189   static const uptr MapSizeIncrement = 1UL << 17;
  192     uptr PoppedBlocks;
  193     uptr PushedBlocks;
  197     uptr PushedBlocksAtLastRelease;
  198     uptr RangesReleased;
  199     uptr LastReleasedBytes;
  210     uptr RegionBeg;
  211     uptr MappedUser;    // Bytes mapped for user memory.
  212     uptr AllocatedUser; // Bytes allocated for user memory.
  218   uptr PrimaryBase;
  223   RegionInfo *getRegionInfo(uptr ClassId) const {
  228   uptr getRegionBaseByClassId(uptr ClassId) const {
  228   uptr getRegionBaseByClassId(uptr ClassId) const {
  232   bool populateBatches(CacheT *C, RegionInfo *Region, uptr ClassId,
  239     for (uptr I = 0; I < Count; I++) {
  256   NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
  258     const uptr Size = getSizeByClassId(ClassId);
  261     const uptr RegionBeg = Region->RegionBeg;
  262     const uptr MappedUser = Region->MappedUser;
  263     const uptr TotalUserBytes = Region->AllocatedUser + MaxCount * Size;
  267       const uptr UserMapSize =
  269       const uptr RegionBase = RegionBeg - getRegionBaseByClassId(ClassId);
  292     const uptr NumberOfBlocks = Min(
  297     constexpr uptr ShuffleArraySize = 48;
  300     const uptr P = RegionBeg + Region->AllocatedUser;
  301     const uptr AllocatedUser = NumberOfBlocks * Size;
  302     for (uptr I = P; I < P + AllocatedUser; I += Size) {
  328   void getStats(ScopedString *Str, uptr ClassId, uptr Rss) const {
  328   void getStats(ScopedString *Str, uptr ClassId, uptr Rss) const {
  332     const uptr InUse = Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks;
  333     const uptr TotalChunks = Region->AllocatedUser / getSizeByClassId(ClassId);
  345   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
  345   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
  347     const uptr BlockSize = getSizeByClassId(ClassId);
  348     const uptr PageSize = getPageSizeCached();
  351     const uptr BytesInFreeList =

   23   uptr Size;
   27   void init(void *Ptr, uptr Size) {
   34   uptr getQuarantinedSize() const { return Size - sizeof(QuarantineBatch); }
   36   void push_back(void *Ptr, uptr Size) {
   50     for (uptr I = 0; I < From->Count; ++I)
   74   uptr getSize() const { return atomic_load_relaxed(&Size); }
   76   uptr getOverheadSize() const { return List.size() * sizeof(QuarantineBatch); }
   78   void enqueue(Callback Cb, void *Ptr, uptr Size) {
  112     uptr ExtractedSize = 0;
  134     uptr BatchCount = 0;
  135     uptr TotalOverheadBytes = 0;
  136     uptr TotalBytes = 0;
  137     uptr TotalQuarantineChunks = 0;
  144     const uptr QuarantineChunksCapacity =
  146     const uptr ChunksUsagePercent =
  150     const uptr TotalQuarantinedBytes = TotalBytes - TotalOverheadBytes;
  151     const uptr MemoryOverheadPercent =
  166   void addToSize(uptr add) { atomic_store_relaxed(&Size, getSize() + add); }
  167   void subFromSize(uptr sub) { atomic_store_relaxed(&Size, getSize() - sub); }
  178   void initLinkerInitialized(uptr Size, uptr CacheSize) {
  178   void initLinkerInitialized(uptr Size, uptr CacheSize) {
  189   void init(uptr Size, uptr CacheSize) {
  189   void init(uptr Size, uptr CacheSize) {
  194   uptr getMaxSize() const { return atomic_load_relaxed(&MaxSize); }
  195   uptr getCacheSize() const { return atomic_load_relaxed(&MaxCacheSize); }
  197   void put(CacheT *C, Callback Cb, Node *Ptr, uptr Size) {
  237   void NOINLINE recycle(uptr MinSize, Callback Cb) {
  247       const uptr CacheSize = Cache.getSize();
  248       const uptr OverheadSize = Cache.getOverheadSize();
  253       constexpr uptr OverheadThresholdPercents = 100;
  273       constexpr uptr NumberOfPrefetch = 8UL;
  275       for (uptr I = 0; I < NumberOfPrefetch; I++)
  277       for (uptr I = 0, Count = B->Count; I < Count; I++) {

   19   ReleaseRecorder(uptr BaseAddress, MapPlatformData *Data = nullptr)
   22   uptr getReleasedRangesCount() const { return ReleasedRangesCount; }
   24   uptr getReleasedBytes() const { return ReleasedBytes; }
   27   void releasePageRangeToOS(uptr From, uptr To) {
   27   void releasePageRangeToOS(uptr From, uptr To) {
   28     const uptr Size = To - From;
   35   uptr ReleasedRangesCount = 0;
   36   uptr ReleasedBytes = 0;
   37   uptr BaseAddress = 0;
   49   PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) {
   49   PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) {
   52     constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL;
   55     const uptr CounterSizeBits =
   61     const uptr PackingRatio = MaxCounterBits >> CounterSizeBitsLog;
   79   uptr getCount() const { return N; }
   81   uptr get(uptr I) const {
   81   uptr get(uptr I) const {
   83     const uptr Index = I >> PackingRatioLog;
   84     const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
   88   void inc(uptr I) const {
   90     const uptr Index = I >> PackingRatioLog;
   91     const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
   96   void incRange(uptr From, uptr To) const {
   96   void incRange(uptr From, uptr To) const {
   98     for (uptr I = From; I <= To; I++)
  102   uptr getBufferSize() const { return BufferSize; }
  105   const uptr N;
  106   uptr CounterSizeBitsLog;
  107   uptr CounterMask;
  108   uptr PackingRatioLog;
  109   uptr BitOffsetMask;
  111   uptr BufferSize;
  112   uptr *Buffer;
  144   const uptr PageSizeLog;
  146   uptr CurrentPage = 0;
  147   uptr CurrentRangeStatePage = 0;
  152 releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
  153                       uptr AllocatedPagesCount, uptr BlockSize,
  153                       uptr AllocatedPagesCount, uptr BlockSize,
  155   const uptr PageSize = getPageSizeCached();
  159   uptr FullPagesBlockCountMax;
  195   const uptr PageSizeLog = getLog2(PageSize);
  196   const uptr End = Base + AllocatedPagesCount * PageSize;
  204         const uptr P = reinterpret_cast<uptr>(It.get(I));
  213         const uptr P = reinterpret_cast<uptr>(It.get(I));
  226     for (uptr I = 0; I < Counters.getCount(); I++)
  231     const uptr Pn = BlockSize < PageSize ? PageSize / BlockSize : 1;
  232     const uptr Pnc = Pn * BlockSize;
  238     uptr PrevPageBoundary = 0;
  239     uptr CurrentBoundary = 0;
  240     for (uptr I = 0; I < Counters.getCount(); I++) {
  241       const uptr PageBoundary = PrevPageBoundary + PageSize;
  242       uptr BlocksPerPage = Pn;

   87 void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment) {
   87 void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment) {
   95 void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize,
   95 void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize,
   96                                          uptr MaxSize) {
  103 void NORETURN reportOutOfMemory(uptr RequestedSize) {
  147 void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size,
  148                                        uptr ExpectedSize) {
  155 void NORETURN reportAlignmentNotPowerOfTwo(uptr Alignment) {
  162 void NORETURN reportCallocOverflow(uptr Count, uptr Size) {
  162 void NORETURN reportCallocOverflow(uptr Count, uptr Size) {
  169 void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment) {
  177 void NORETURN reportPvallocOverflow(uptr Size) {
  184 void NORETURN reportInvalidAlignedAllocAlignment(uptr Alignment, uptr Size) {
  184 void NORETURN reportInvalidAlignedAllocAlignment(uptr Alignment, uptr Size) {

   32 void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment);
   32 void NORETURN reportAlignmentTooBig(uptr Alignment, uptr MaxAlignment);
   33 void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize,
   33 void NORETURN reportAllocationSizeTooBig(uptr UserSize, uptr TotalSize,
   34                                          uptr MaxSize);
   35 void NORETURN reportOutOfMemory(uptr RequestedSize);
   46 void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size, uptr ExpectedSize);
   46 void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size, uptr ExpectedSize);
   49 void NORETURN reportAlignmentNotPowerOfTwo(uptr Alignment);
   50 void NORETURN reportInvalidPosixMemalignAlignment(uptr Alignment);
   51 void NORETURN reportCallocOverflow(uptr Count, uptr Size);
   51 void NORETURN reportCallocOverflow(uptr Count, uptr Size);
   52 void NORETURN reportPvallocOverflow(uptr Size);
   53 void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment);
   53 void NORETURN reportInvalidAlignedAllocAlignment(uptr Size, uptr Alignment);

   26 void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, uptr *BlockEnd) {
   26 void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, uptr *BlockEnd) {
   26 void *MapAllocator::allocate(uptr Size, uptr AlignmentHint, uptr *BlockEnd) {
   28   const uptr PageSize = getPageSizeCached();
   29   const uptr MapSize =
   32   uptr MapBase =
   37   uptr CommitBase = MapBase + PageSize;
   38   uptr MapEnd = MapBase + MapSize;
   47     const uptr NewMapBase = CommitBase - PageSize;
   55     const uptr NewMapEnd = CommitBase + PageSize +
   65   const uptr CommitSize = MapEnd - PageSize - CommitBase;
   66   const uptr Ptr =
   94     const uptr CommitSize = H->BlockEnd - reinterpret_cast<uptr>(H);
  101   const uptr Size = H->MapSize;

   31   uptr BlockEnd;
   32   uptr MapBase;
   33   uptr MapSize;
   37 constexpr uptr getHeaderSize() {
   41 static Header *getHeader(uptr Ptr) {
   63   void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr);
   63   void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr);
   63   void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr);
   67   static uptr getBlockEnd(void *Ptr) {
   71   static uptr getBlockSize(void *Ptr) {
   89   uptr AllocatedBytes;
   90   uptr FreedBytes;
   91   uptr LargestSize;

   40   static const uptr MinSize = 1UL << MinSizeLog;
   41   static const uptr MidSize = 1UL << MidSizeLog;
   42   static const uptr MidClass = MidSize / MinSize;
   44   static const uptr M = (1UL << S) - 1;
   49   static const uptr MaxSize = 1UL << MaxSizeLog;
   50   static const uptr NumClasses =
   53   static const uptr LargestClassId = NumClasses - 1;
   54   static const uptr BatchClassId = 0;
   56   static uptr getSizeByClassId(uptr ClassId) {
   56   static uptr getSizeByClassId(uptr ClassId) {
   61     const uptr T = MidSize << (ClassId >> S);
   65   static uptr getClassIdBySize(uptr Size) {
   65   static uptr getClassIdBySize(uptr Size) {
   69     const uptr L = getMostSignificantSetBitIndex(Size);
   70     const uptr HBits = (Size >> (L - S)) & M;
   71     const uptr LBits = Size & ((1UL << (L - S)) - 1);
   72     const uptr L1 = L - MidSizeLog;
   76   static u32 getMaxCachedHint(uptr Size) {
   90     uptr PrevS = 0;
   91     uptr TotalCached = 0;
   92     for (uptr I = 0; I < NumClasses; I++) {
   95       const uptr S = getSizeByClassId(I);
   98       const uptr D = S - PrevS;
   99       const uptr P = PrevS ? (D * 100 / PrevS) : 0;
  100       const uptr L = S ? getMostSignificantSetBitIndex(S) : 0;
  101       const uptr Cached = getMaxCachedHint(S) * S;
  114     for (uptr C = 0; C < NumClasses; C++) {
  117       const uptr S = getSizeByClassId(C);
  128     for (uptr S = 1; S <= MaxSize; S++) {
  129       const uptr C = getClassIdBySize(S);

   23 typedef uptr StatCounters[StatCount];
   35   void add(StatType I, uptr V) {
   40   void sub(StatType I, uptr V) {
   45   void set(StatType I, uptr V) { atomic_store_relaxed(&StatsArray[I], V); }
   47   uptr get(StatType I) const { return atomic_load_relaxed(&StatsArray[I]); }
   73     for (uptr I = 0; I < StatCount; I++)
   77   void get(uptr *S) const {
   79     for (uptr I = 0; I < StatCount; I++)
   82       for (uptr I = 0; I < StatCount; I++)
   86     for (uptr I = 0; I < StatCount; I++)

   31   constexpr uptr MaxLen = 30;
   41   uptr NumBuffer[MaxLen];
  117 int formatString(char *Buffer, uptr BufferLength, const char *Format,
  218   const uptr AdditionalLength =

   21   explicit ScopedString(uptr MaxLength) : String(MaxLength), Length(0) {
   24   uptr length() { return Length; }
   36   uptr Length;

   57   INLINE uptr getPrecedence() { return atomic_load_relaxed(&Precedence); }

  118       uptr LowestPrecedence = UINTPTR_MAX;
  126         const uptr Precedence = TSDs[Index].getPrecedence();

   22   void init(uptr InitialCapacity) {
   32   T &operator[](uptr I) {
   36   const T &operator[](uptr I) const {
   43       const uptr NewCapacity = roundUpToPowerOfTwo(Size + 1);
   56   uptr size() const { return Size; }
   59   uptr capacity() const { return CapacityBytes / sizeof(T); }
   60   void reserve(uptr NewSize) {
   65   void resize(uptr NewSize) {
   82   void reallocate(uptr NewCapacity) {
   85     const uptr NewCapacityBytes =
   97   uptr CapacityBytes;
   98   uptr Size;
  104   explicit Vector(uptr Count) {

   21   scudo::uptr Product;
  106   const scudo::uptr PageSize = scudo::getPageSizeCached();

   33 INLINE bool checkAlignedAllocAlignmentAndSize(uptr Alignment, uptr Size) {
   33 INLINE bool checkAlignedAllocAlignmentAndSize(uptr Alignment, uptr Size) {
   40 INLINE bool checkPosixMemalignAlignment(uptr Alignment) {
   48 INLINE bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) {
   48 INLINE bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) {
   48 INLINE bool checkForCallocOverflow(uptr Size, uptr N, uptr *Product) {
   61 INLINE bool checkForPvallocOverflow(uptr Size, uptr PageSize) {
   61 INLINE bool checkForPvallocOverflow(uptr Size, uptr PageSize) {