//! Memory allocator /* * This file incorporates work covered by the following license notice: * * Copyright (c) 2020, the SerenityOS developers. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use core::{ alloc::{GlobalAlloc, Layout}, mem, ptr::{self, NonNull}, }; use spin::Mutex; extern "C" { fn _initial_kernel_heap_start(); fn _initial_kernel_heap_size(); } const INITIAL_KERNEL_HEAP_START: *mut u8 = _initial_kernel_heap_start as _; const INITIAL_KERNEL_HEAP_SIZE: *const () = _initial_kernel_heap_size as _; struct Allocator(Mutex>); unsafe impl GlobalAlloc for Allocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { let mut lock = self.0.lock(); let allocator = lock.as_mut().expect("heap allocator should be initialized"); match allocator.allocate(layout.size(), layout.align()) { Some(ptr) => ptr.as_ptr(), None => ptr::null_mut(), } } unsafe fn dealloc(&self, ptr: *mut u8, _: Layout) { let mut lock = self.0.lock(); let allocator = lock.as_mut().expect("heap allocator should be initialized"); allocator.deallocate(ptr); } } #[global_allocator] static ALLOCATOR: Allocator = Allocator(Mutex::new(None)); pub fn init() { log::info!("Initialising kernel heap allocator"); *ALLOCATOR.0.lock() = Some(unsafe { Heap::new(INITIAL_KERNEL_HEAP_START, INITIAL_KERNEL_HEAP_SIZE as _) }); } // FIXME: these are arch-specific const CHUNK_SIZE: usize = 16; const MINIMUM_ALIGNMENT: usize = 8; struct Header { size_in_chunks: usize, } // compile-time assertions to make sure that AllocationHeader's size is a power of two // and CHUNK_SIZE is bigger than AllocationHeader's size const _: () = { assert!(mem::size_of::
().is_power_of_two()); assert!(CHUNK_SIZE >= mem::size_of::
()); }; /// A first-fit heap allocator, with CHUNK_SIZE chunks and a set size /// In the future these will become subheaps and the actual heap allocator will create more /// subheaps as needed struct Heap { total_chunks: usize, allocated_chunks: usize, chunks: *mut u8, bitmap: *mut u8, } impl Heap { /// # Safety /// This function assumes that the pointer given points at a valid memory address unsafe fn new(memory: *mut u8, memory_size: usize) -> Self { let total_chunks = Self::calculate_chunks(memory_size); assert!(total_chunks * CHUNK_SIZE + (total_chunks + 7) / 8 <= memory_size); Self { total_chunks, allocated_chunks: 0, chunks: memory, bitmap: unsafe { memory.add(total_chunks * CHUNK_SIZE) }, } } fn allocate(&mut self, size: usize, alignment: usize) -> Option> { assert!(alignment.is_power_of_two()); let alignment = if alignment < MINIMUM_ALIGNMENT { MINIMUM_ALIGNMENT } else { alignment }; // We need space for the header as well let size = size + mem::size_of::
(); let chunks_needed = (size + CHUNK_SIZE - 1) / CHUNK_SIZE; let chunk_alignment = (alignment + CHUNK_SIZE - 1) / CHUNK_SIZE; log::debug!("size: {size} chunks: {chunks_needed} align: {chunk_alignment}"); if chunks_needed + chunk_alignment > self.free_chunks() { return None; } // FIXME: should utilize the alignment directly instead of trying to allocate `size + alignment` let first_chunk = self.find_first_fit(chunks_needed + chunk_alignment)?; let chunks_addr = self.chunks as usize; let addr_unaligned = chunks_addr + first_chunk * CHUNK_SIZE; // Align the starting address and verify that we haven't gone outside the calculated free area let addr = addr_unaligned + alignment - (addr_unaligned + mem::size_of::
()) % alignment; log::debug!( "Addr unaligned: 0x{addr_unaligned:x} (offset: 0x{:x})", addr_unaligned - chunks_addr ); log::trace!("Addr: 0x{addr:x} (offset: 0x{:x})", addr - chunks_addr); let aligned_first_chunk = (addr - chunks_addr) / CHUNK_SIZE; assert!(first_chunk <= aligned_first_chunk); assert!( aligned_first_chunk + chunks_needed <= first_chunk + chunks_needed + chunk_alignment ); let header: *mut Header = unsafe { mem::transmute(addr) }; unsafe { (*header).size_in_chunks = chunks_needed; } self.bitmap_set_range(aligned_first_chunk, chunks_needed, true); self.allocated_chunks += chunks_needed; let ptr: *mut u8 = unsafe { mem::transmute(header.add(1)) }; log::trace!("{ptr:p}"); // FIXME: zero or scrub memory? assert!(ptr.is_aligned_to(alignment)); NonNull::new(ptr) } fn deallocate(&mut self, ptr: *mut u8) { let header = Self::allocation_header(ptr); let start = (header as usize - self.chunks as usize) / CHUNK_SIZE; assert!(self.bitmap_get(start)); let size = unsafe { (*header).size_in_chunks }; self.bitmap_set_range(start, size, false); self.allocated_chunks -= size; // FIXME: zero or scrub memory? } /// Finds first hole that can fit an allocation of `size` chunks, returns the start of the /// found free chunks fn find_first_fit(&self, size: usize) -> Option { let mut start_of_free_chunks = 0; let mut free_chunks = 0; for i in 0..self.total_chunks / usize::BITS as usize { if free_chunks >= size { return Some(start_of_free_chunks); } let mut bucket = unsafe { *self.bitmap.cast::().add(i) }; if bucket == usize::MAX { // Skip over completely full bucket free_chunks = 0; continue; } if bucket == 0 { // Skip over completely empty bucket if free_chunks == 0 { start_of_free_chunks = i * usize::BITS as usize; } free_chunks += usize::BITS as usize; continue; } let mut viewed_bits = 0; while viewed_bits < usize::BITS as usize { if bucket == 0 { if free_chunks == 0 { start_of_free_chunks = i * usize::BITS as usize + viewed_bits; } free_chunks += usize::BITS as usize - viewed_bits; viewed_bits = usize::BITS as usize; } else { let trailing_zeros = bucket.trailing_zeros() as usize; bucket >>= trailing_zeros; if free_chunks == 0 { start_of_free_chunks = i * usize::BITS as usize + viewed_bits; } free_chunks += trailing_zeros; viewed_bits += trailing_zeros; if free_chunks >= size { return Some(start_of_free_chunks); } let trailing_ones = bucket.trailing_ones() as usize; bucket >>= trailing_ones; viewed_bits += trailing_ones; free_chunks = 0; } } } if free_chunks >= size { return Some(start_of_free_chunks); } let first_trailing_bit = (self.total_chunks / usize::BITS as usize) * usize::BITS as usize; let trailing_bits = self.total_chunks % usize::BITS as usize; for i in 0..trailing_bits { if self.bitmap_get(first_trailing_bit + i) { free_chunks = 0; continue; } if free_chunks == 0 { start_of_free_chunks = first_trailing_bit + i; } free_chunks += 1; if free_chunks >= size { return Some(start_of_free_chunks); } } None } fn bitmap_set_range(&mut self, start: usize, length: usize, value: bool) { assert!(start + length <= self.total_chunks); if length == 0 { return; } const BITMASK_FIRST_BYTE: [u8; 8] = [0xFF, 0xFE, 0xFC, 0xF8, 0xF0, 0xE0, 0xC0, 0x80]; const BITMASK_LAST_BYTE: [u8; 8] = [0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F]; let first = unsafe { self.bitmap.add(start / 8) }; let last = unsafe { self.bitmap.add((start + length) / 8) }; let mut byte_mask = BITMASK_FIRST_BYTE[start % 8]; if first == last { byte_mask &= BITMASK_LAST_BYTE[(start + length) % 8]; if value { unsafe { *first |= byte_mask; } } else { unsafe { *first &= !byte_mask; } } } else { if value { unsafe { *first |= byte_mask; } } else { unsafe { *first &= !byte_mask; } } byte_mask = BITMASK_LAST_BYTE[(start + length) % 8]; if value { unsafe { *last |= byte_mask; } } else { unsafe { *last &= !byte_mask; } } let first = unsafe { first.add(1) }; if first >= last { return; } if value { unsafe { first.write_bytes(0xFF, last.sub_ptr(first)); } } else { unsafe { first.write_bytes(0, last.sub_ptr(first)); } } } } fn bitmap_get(&self, index: usize) -> bool { assert!(index < self.total_chunks); (unsafe { *self.bitmap.add(index / 8) } & (1 << (index % 8))) != 0 } const fn free_chunks(&self) -> usize { self.total_chunks - self.allocated_chunks } fn allocation_header(ptr: *mut u8) -> *mut Header { unsafe { mem::transmute::<_, *mut Header>(ptr).sub(1) } } const fn calculate_chunks(memory_size: usize) -> usize { memory_size / (CHUNK_SIZE + 1) } } unsafe impl Send for Heap {} #[alloc_error_handler] fn alloc_error_handler(layout: alloc::alloc::Layout) -> ! { loop {} }