1 use alloc::vec::Vec; 2 use core::{arch::asm, mem, ptr, slice}; 3 use goblin::error::{Error, Result}; 4 5 use super::ExpectTlsFree; 6 use crate::{ 7 header::sys_mman, 8 ld_so::linker::Linker, 9 platform::{Pal, Sys}, 10 sync::mutex::Mutex, 11 }; 12 13 #[repr(C)] 14 #[derive(Debug)] 15 pub struct Master { 16 /// Pointer to initial data 17 pub ptr: *const u8, 18 /// Length of initial data in bytes 19 pub len: usize, 20 /// Offset in TLS to copy initial data to 21 pub offset: usize, 22 } 23 24 impl Master { 25 /// The initial data for this TLS region 26 pub unsafe fn data(&self) -> &'static [u8] { 27 slice::from_raw_parts(self.ptr, self.len) 28 } 29 } 30 31 #[derive(Debug)] 32 #[repr(C)] 33 pub struct Tcb { 34 /// Pointer to the end of static TLS. Must be the first member 35 pub tls_end: *mut u8, 36 /// Size of the memory allocated for the static TLS in bytes (multiple of page size) 37 pub tls_len: usize, 38 /// Pointer to this structure 39 pub tcb_ptr: *mut Tcb, 40 /// Size of the memory allocated for this structure in bytes (should be same as page size) 41 pub tcb_len: usize, 42 /// Pointer to a list of initial TLS data 43 pub masters_ptr: *mut Master, 44 /// Size of the masters list in bytes (multiple of mem::size_of::<Master>()) 45 pub masters_len: usize, 46 /// Index of last copied Master 47 pub num_copied_masters: usize, 48 /// Pointer to dynamic linker 49 pub linker_ptr: *const Mutex<Linker>, 50 /// pointer to rust memory allocator structure 51 pub mspace: usize, 52 } 53 54 impl Tcb { 55 /// Create a new TCB 56 pub unsafe fn new(size: usize) -> Result<&'static mut Self> { 57 let page_size = Sys::getpagesize(); 58 let (abi_page, tls, tcb_page) = Self::os_new(round_up(size, page_size))?; 59 60 let tcb_ptr = tcb_page.as_mut_ptr() as *mut Self; 61 trace!("New TCB: {:p}", tcb_ptr); 62 ptr::write( 63 tcb_ptr, 64 Self { 65 tls_end: tls.as_mut_ptr().add(tls.len()), 66 tls_len: tls.len(), 67 tcb_ptr, 68 tcb_len: tcb_page.len(), 69 masters_ptr: ptr::null_mut(), 70 masters_len: 0, 71 num_copied_masters: 0, 72 linker_ptr: ptr::null(), 73 mspace: 0, 74 }, 75 ); 76 77 Ok(&mut *tcb_ptr) 78 } 79 80 /// Get the current TCB 81 pub unsafe fn current() -> Option<&'static mut Self> { 82 let tcb_ptr = Self::arch_read(offset_of!(Self, tcb_ptr)) as *mut Self; 83 let tcb_len = Self::arch_read(offset_of!(Self, tcb_len)); 84 if tcb_ptr.is_null() || tcb_len < mem::size_of::<Self>() { 85 None 86 } else { 87 Some(&mut *tcb_ptr) 88 } 89 } 90 91 /// A slice for all of the TLS data 92 pub unsafe fn tls(&self) -> Option<&'static mut [u8]> { 93 if self.tls_end.is_null() || self.tls_len == 0 { 94 None 95 } else { 96 Some(slice::from_raw_parts_mut( 97 self.tls_end.offset(-(self.tls_len as isize)), 98 self.tls_len, 99 )) 100 } 101 } 102 103 /// The initial images for TLS 104 pub unsafe fn masters(&self) -> Option<&'static mut [Master]> { 105 if self.masters_ptr.is_null() || self.masters_len == 0 { 106 None 107 } else { 108 Some(slice::from_raw_parts_mut( 109 self.masters_ptr, 110 self.masters_len / mem::size_of::<Master>(), 111 )) 112 } 113 } 114 115 /// Copy data from masters 116 pub unsafe fn copy_masters(&mut self) -> Result<()> { 117 //TODO: Complain if masters or tls exist without the other 118 if let Some(tls) = self.tls() { 119 if let Some(masters) = self.masters() { 120 for (i, master) in masters 121 .iter() 122 .skip(self.num_copied_masters) 123 .filter(|m| m.len > 0) 124 .enumerate() 125 { 126 let range = if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { 127 // x86 TLS layout is backwards 128 self.tls_len - master.offset..self.tls_len - master.offset + master.len 129 } else { 130 //TODO: fix aarch64 TLS layout when there is more than one master 131 assert_eq!(i, 0, "aarch64 TLS layout only supports one master"); 132 0..master.len 133 }; 134 if let Some(tls_data) = tls.get_mut(range) { 135 let data = master.data(); 136 trace!( 137 "tls master {}: {:p}, {:#x}: {:p}, {:#x}", 138 i, 139 data.as_ptr(), 140 data.len(), 141 tls_data.as_mut_ptr(), 142 tls_data.len() 143 ); 144 tls_data.copy_from_slice(data); 145 } else { 146 return Err(Error::Malformed(format!("failed to copy tls master {}", i))); 147 } 148 } 149 self.num_copied_masters = masters.len(); 150 } 151 } 152 153 Ok(()) 154 } 155 156 /// The initial images for TLS 157 pub unsafe fn append_masters(&mut self, mut new_masters: Vec<Master>) { 158 if self.masters_ptr.is_null() { 159 self.masters_ptr = new_masters.as_mut_ptr(); 160 self.masters_len = new_masters.len() * mem::size_of::<Master>(); 161 mem::forget(new_masters); 162 } else { 163 let len = self.masters_len / mem::size_of::<Master>(); 164 let mut masters = Vec::from_raw_parts(self.masters_ptr, len, len); 165 masters.extend(new_masters.into_iter()); 166 self.masters_ptr = masters.as_mut_ptr(); 167 self.masters_len = masters.len() * mem::size_of::<Master>(); 168 mem::forget(masters); 169 } 170 } 171 172 /// Activate TLS 173 pub unsafe fn activate(&mut self) { 174 Self::os_arch_activate(self.tls_end as usize, self.tls_len); 175 } 176 177 /// Mapping with correct flags for TCB and TLS 178 unsafe fn map(size: usize) -> Result<&'static mut [u8]> { 179 let ptr = sys_mman::mmap( 180 ptr::null_mut(), 181 size, 182 sys_mman::PROT_READ | sys_mman::PROT_WRITE, 183 sys_mman::MAP_ANONYMOUS | sys_mman::MAP_PRIVATE, 184 -1, 185 0, 186 ); 187 if ptr as usize == !0 188 /* MAP_FAILED */ 189 { 190 return Err(Error::Malformed(format!("failed to map tls"))); 191 } 192 ptr::write_bytes(ptr as *mut u8, 0, size); 193 Ok(slice::from_raw_parts_mut(ptr as *mut u8, size)) 194 } 195 196 /// OS specific code to create a new TLS and TCB - Linux and Redox 197 #[cfg(any(target_os = "linux", target_os = "redox"))] 198 unsafe fn os_new( 199 size: usize, 200 ) -> Result<(&'static mut [u8], &'static mut [u8], &'static mut [u8])> { 201 let page_size = Sys::getpagesize(); 202 let abi_tls_tcb = Self::map(page_size + size + page_size)?; 203 let (abi, tls_tcb) = abi_tls_tcb.split_at_mut(page_size); 204 let (tls, tcb) = tls_tcb.split_at_mut(size); 205 Ok((abi, tls, tcb)) 206 } 207 208 /// Architecture specific code to read a usize from the TCB - aarch64 209 #[inline(always)] 210 #[cfg(target_arch = "aarch64")] 211 unsafe fn arch_read(offset: usize) -> usize { 212 let abi_ptr: usize; 213 asm!( 214 "mrs {}, tpidr_el0", 215 out(reg) abi_ptr, 216 ); 217 218 let tcb_ptr = *(abi_ptr as *const usize); 219 *((tcb_ptr + offset) as *const usize) 220 } 221 222 /// Architecture specific code to read a usize from the TCB - x86 223 #[inline(always)] 224 #[cfg(target_arch = "x86")] 225 unsafe fn arch_read(offset: usize) -> usize { 226 let value; 227 asm!( 228 " 229 mov {}, gs:[{}] 230 ", 231 out(reg) value, 232 in(reg) offset, 233 ); 234 value 235 } 236 237 /// Architecture specific code to read a usize from the TCB - x86_64 238 #[inline(always)] 239 #[cfg(target_arch = "x86_64")] 240 unsafe fn arch_read(offset: usize) -> usize { 241 let value; 242 asm!( 243 " 244 mov {}, fs:[{}] 245 ", 246 out(reg) value, 247 in(reg) offset, 248 ); 249 value 250 } 251 252 /// OS and architecture specific code to activate TLS - Linux x86_64 253 #[cfg(all(target_os = "linux", target_arch = "x86_64"))] 254 unsafe fn os_arch_activate(tls_end: usize, _tls_len: usize) { 255 const ARCH_SET_FS: usize = 0x1002; 256 syscall!(ARCH_PRCTL, ARCH_SET_FS, tls_end); 257 } 258 259 /// OS and architecture specific code to activate TLS - Redox aarch64 260 #[cfg(all(target_os = "redox", target_arch = "aarch64"))] 261 unsafe fn os_arch_activate(tls_end: usize, tls_len: usize) { 262 // Uses ABI page 263 let abi_ptr = tls_end - tls_len - 16; 264 ptr::write(abi_ptr as *mut usize, tls_end); 265 asm!( 266 "msr tpidr_el0, {}", 267 in(reg) abi_ptr, 268 ); 269 } 270 271 /// OS and architecture specific code to activate TLS - Redox x86 272 #[cfg(all(target_os = "redox", target_arch = "x86"))] 273 unsafe fn os_arch_activate(tls_end: usize, _tls_len: usize) { 274 let mut env = syscall::EnvRegisters::default(); 275 276 let file = syscall::open( 277 "thisproc:current/regs/env", 278 syscall::O_CLOEXEC | syscall::O_RDWR, 279 ) 280 .expect_notls("failed to open handle for process registers"); 281 282 let _ = syscall::read(file, &mut env).expect_notls("failed to read gsbase"); 283 284 env.gsbase = tls_end as u32; 285 286 let _ = syscall::write(file, &env).expect_notls("failed to write gsbase"); 287 288 let _ = syscall::close(file); 289 } 290 291 /// OS and architecture specific code to activate TLS - Redox x86_64 292 #[cfg(all(target_os = "redox", target_arch = "x86_64"))] 293 unsafe fn os_arch_activate(tls_end: usize, _tls_len: usize) { 294 let mut env = syscall::EnvRegisters::default(); 295 296 let file = syscall::open( 297 "thisproc:current/regs/env", 298 syscall::O_CLOEXEC | syscall::O_RDWR, 299 ) 300 .expect_notls("failed to open handle for process registers"); 301 302 let _ = syscall::read(file, &mut env).expect_notls("failed to read fsbase"); 303 304 env.fsbase = tls_end as u64; 305 306 let _ = syscall::write(file, &env).expect_notls("failed to write fsbase"); 307 308 let _ = syscall::close(file); 309 } 310 } 311 312 pub fn round_up(value: usize, alignment: usize) -> usize { 313 return (value + alignment - 1) & (!(alignment - 1)); 314 } 315