1use std::error;
14use std::fmt;
15use std::io;
16use std::os::unix::io::AsRawFd;
17use std::ptr::null_mut;
18use std::result;
19
20use crate::bitmap::{Bitmap, BS};
21use crate::guest_memory::FileOffset;
22use crate::mmap::{check_file_offset, AsSlice, NewBitmap};
23use crate::volatile_memory::{self, compute_offset, VolatileMemory, VolatileSlice};
24
25#[derive(Debug)]
27pub enum Error {
28 InvalidOffsetLength,
30 InvalidPointer,
32 MapFixed,
34 MappingOverlap,
36 MappingPastEof,
38 Mmap(io::Error),
40 SeekEnd(io::Error),
42 SeekStart(io::Error),
44}
45
46impl fmt::Display for Error {
47 fn fmt(&self, f: &mut std::fmt::Formatter) -> fmt::Result {
48 match self {
49 Error::InvalidOffsetLength => write!(
50 f,
51 "The specified file offset and length cause overflow when added"
52 ),
53 Error::InvalidPointer => write!(
54 f,
55 "The specified pointer to the mapping is not page-aligned",
56 ),
57 Error::MapFixed => write!(f, "The forbidden `MAP_FIXED` flag was specified"),
58 Error::MappingOverlap => write!(
59 f,
60 "Mappings using the same fd overlap in terms of file offset and length"
61 ),
62 Error::MappingPastEof => write!(
63 f,
64 "The specified file offset and length is greater then file length"
65 ),
66 Error::Mmap(error) => write!(f, "{}", error),
67 Error::SeekEnd(error) => write!(f, "Error seeking the end of the file: {}", error),
68 Error::SeekStart(error) => write!(f, "Error seeking the start of the file: {}", error),
69 }
70 }
71}
72
73impl error::Error for Error {}
74
75pub type Result<T> = result::Result<T, Error>;
76
77pub struct MmapRegionBuilder<B = ()> {
79 size: usize,
80 prot: i32,
81 flags: i32,
82 file_offset: Option<FileOffset>,
83 raw_ptr: Option<*mut u8>,
84 hugetlbfs: Option<bool>,
85 bitmap: B,
86}
87
88impl<B: Bitmap + Default> MmapRegionBuilder<B> {
89 pub fn new(size: usize) -> Self {
92 Self::new_with_bitmap(size, B::default())
93 }
94}
95
96impl<B: Bitmap> MmapRegionBuilder<B> {
97 pub fn new_with_bitmap(size: usize, bitmap: B) -> Self {
103 MmapRegionBuilder {
104 size,
105 prot: 0,
106 flags: libc::MAP_ANONYMOUS | libc::MAP_PRIVATE,
107 file_offset: None,
108 raw_ptr: None,
109 hugetlbfs: None,
110 bitmap,
111 }
112 }
113
114 pub fn with_mmap_prot(mut self, prot: i32) -> Self {
116 self.prot = prot;
117 self
118 }
119
120 pub fn with_mmap_flags(mut self, flags: i32) -> Self {
122 self.flags = flags;
123 self
124 }
125
126 pub fn with_file_offset(mut self, file_offset: FileOffset) -> Self {
128 self.file_offset = Some(file_offset);
129 self
130 }
131
132 pub fn with_hugetlbfs(mut self, hugetlbfs: bool) -> Self {
134 self.hugetlbfs = Some(hugetlbfs);
135 self
136 }
137
138 pub unsafe fn with_raw_mmap_pointer(mut self, raw_ptr: *mut u8) -> Self {
145 self.raw_ptr = Some(raw_ptr);
146 self
147 }
148
149 pub fn build(self) -> Result<MmapRegion<B>> {
151 if self.raw_ptr.is_some() {
152 return self.build_raw();
153 }
154
155 if self.flags & libc::MAP_FIXED != 0 {
158 return Err(Error::MapFixed);
159 }
160
161 let (fd, offset) = if let Some(ref f_off) = self.file_offset {
162 check_file_offset(f_off, self.size)?;
163 (f_off.file().as_raw_fd(), f_off.start())
164 } else {
165 (-1, 0)
166 };
167
168 let addr = unsafe {
172 libc::mmap(
173 null_mut(),
174 self.size,
175 self.prot,
176 self.flags,
177 fd,
178 offset as libc::off_t,
179 )
180 };
181
182 if addr == libc::MAP_FAILED {
183 return Err(Error::Mmap(io::Error::last_os_error()));
184 }
185
186 Ok(MmapRegion {
187 addr: addr as *mut u8,
188 size: self.size,
189 bitmap: self.bitmap,
190 file_offset: self.file_offset,
191 prot: self.prot,
192 flags: self.flags,
193 owned: true,
194 hugetlbfs: self.hugetlbfs,
195 })
196 }
197
198 fn build_raw(self) -> Result<MmapRegion<B>> {
199 let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
202 let addr = self.raw_ptr.unwrap();
203
204 if (addr as usize) & (page_size - 1) != 0 {
206 return Err(Error::InvalidPointer);
207 }
208
209 Ok(MmapRegion {
210 addr,
211 size: self.size,
212 bitmap: self.bitmap,
213 file_offset: self.file_offset,
214 prot: self.prot,
215 flags: self.flags,
216 owned: false,
217 hugetlbfs: self.hugetlbfs,
218 })
219 }
220}
221
222#[derive(Debug)]
232pub struct MmapRegion<B = ()> {
233 addr: *mut u8,
234 size: usize,
235 bitmap: B,
236 file_offset: Option<FileOffset>,
237 prot: i32,
238 flags: i32,
239 owned: bool,
240 hugetlbfs: Option<bool>,
241}
242
243unsafe impl<B: Send> Send for MmapRegion<B> {}
248unsafe impl<B: Sync> Sync for MmapRegion<B> {}
250
251impl<B: NewBitmap> MmapRegion<B> {
252 pub fn new(size: usize) -> Result<Self> {
257 MmapRegionBuilder::new_with_bitmap(size, B::with_len(size))
258 .with_mmap_prot(libc::PROT_READ | libc::PROT_WRITE)
259 .with_mmap_flags(libc::MAP_ANONYMOUS | libc::MAP_NORESERVE | libc::MAP_PRIVATE)
260 .build()
261 }
262
263 pub fn from_file(file_offset: FileOffset, size: usize) -> Result<Self> {
270 MmapRegionBuilder::new_with_bitmap(size, B::with_len(size))
271 .with_file_offset(file_offset)
272 .with_mmap_prot(libc::PROT_READ | libc::PROT_WRITE)
273 .with_mmap_flags(libc::MAP_NORESERVE | libc::MAP_SHARED)
274 .build()
275 }
276
277 pub fn build(
288 file_offset: Option<FileOffset>,
289 size: usize,
290 prot: i32,
291 flags: i32,
292 ) -> Result<Self> {
293 let mut builder = MmapRegionBuilder::new_with_bitmap(size, B::with_len(size))
294 .with_mmap_prot(prot)
295 .with_mmap_flags(flags);
296 if let Some(v) = file_offset {
297 builder = builder.with_file_offset(v);
298 }
299 builder.build()
300 }
301
302 pub unsafe fn build_raw(addr: *mut u8, size: usize, prot: i32, flags: i32) -> Result<Self> {
320 MmapRegionBuilder::new_with_bitmap(size, B::with_len(size))
321 .with_raw_mmap_pointer(addr)
322 .with_mmap_prot(prot)
323 .with_mmap_flags(flags)
324 .build()
325 }
326}
327
328impl<B: Bitmap> MmapRegion<B> {
329 pub fn as_ptr(&self) -> *mut u8 {
335 self.addr
336 }
337
338 pub fn size(&self) -> usize {
340 self.size
341 }
342
343 pub fn file_offset(&self) -> Option<&FileOffset> {
345 self.file_offset.as_ref()
346 }
347
348 pub fn prot(&self) -> i32 {
350 self.prot
351 }
352
353 pub fn flags(&self) -> i32 {
355 self.flags
356 }
357
358 pub fn owned(&self) -> bool {
360 self.owned
361 }
362
363 pub fn fds_overlap<T: Bitmap>(&self, other: &MmapRegion<T>) -> bool {
369 if let Some(f_off1) = self.file_offset() {
370 if let Some(f_off2) = other.file_offset() {
371 if f_off1.file().as_raw_fd() == f_off2.file().as_raw_fd() {
372 let s1 = f_off1.start();
373 let s2 = f_off2.start();
374 let l1 = self.len() as u64;
375 let l2 = other.len() as u64;
376
377 if s1 < s2 {
378 return s1 + l1 > s2;
379 } else {
380 return s2 + l2 > s1;
381 }
382 }
383 }
384 }
385 false
386 }
387
388 pub fn set_hugetlbfs(&mut self, hugetlbfs: bool) {
390 self.hugetlbfs = Some(hugetlbfs)
391 }
392
393 pub fn is_hugetlbfs(&self) -> Option<bool> {
395 self.hugetlbfs
396 }
397
398 pub fn bitmap(&self) -> &B {
400 &self.bitmap
401 }
402}
403
404impl<B> AsSlice for MmapRegion<B> {
405 unsafe fn as_slice(&self) -> &[u8] {
408 std::slice::from_raw_parts(self.addr, self.size)
409 }
410
411 #[allow(clippy::mut_from_ref)]
414 unsafe fn as_mut_slice(&self) -> &mut [u8] {
415 std::slice::from_raw_parts_mut(self.addr, self.size)
416 }
417}
418
419impl<B: Bitmap> VolatileMemory for MmapRegion<B> {
420 type B = B;
421
422 fn len(&self) -> usize {
423 self.size
424 }
425
426 fn get_slice(
427 &self,
428 offset: usize,
429 count: usize,
430 ) -> volatile_memory::Result<VolatileSlice<BS<B>>> {
431 let end = compute_offset(offset, count)?;
432 if end > self.size {
433 return Err(volatile_memory::Error::OutOfBounds { addr: end });
434 }
435
436 Ok(
437 unsafe {
440 VolatileSlice::with_bitmap(
441 (self.addr as usize + offset) as *mut _,
442 count,
443 self.bitmap.slice_at(offset),
444 )
445 },
446 )
447 }
448}
449
450impl<B> Drop for MmapRegion<B> {
451 fn drop(&mut self) {
452 if self.owned {
453 unsafe {
456 libc::munmap(self.addr as *mut libc::c_void, self.size);
457 }
458 }
459 }
460}
461
462#[cfg(test)]
463mod tests {
464 #![allow(clippy::undocumented_unsafe_blocks)]
465 use super::*;
466
467 use std::io::Write;
468 use std::slice;
469 use std::sync::Arc;
470 use vmm_sys_util::tempfile::TempFile;
471
472 use crate::bitmap::AtomicBitmap;
473
474 type MmapRegion = super::MmapRegion<()>;
475
476 impl Error {
479 pub fn raw_os_error(&self) -> i32 {
480 match self {
481 Error::Mmap(e) => e.raw_os_error().unwrap(),
482 _ => std::i32::MIN,
483 }
484 }
485 }
486
487 #[test]
488 fn test_mmap_region_new() {
489 assert!(MmapRegion::new(0).is_err());
490
491 let size = 4096;
492
493 let r = MmapRegion::new(4096).unwrap();
494 assert_eq!(r.size(), size);
495 assert!(r.file_offset().is_none());
496 assert_eq!(r.prot(), libc::PROT_READ | libc::PROT_WRITE);
497 assert_eq!(
498 r.flags(),
499 libc::MAP_ANONYMOUS | libc::MAP_NORESERVE | libc::MAP_PRIVATE
500 );
501 }
502
503 #[test]
504 fn test_mmap_region_set_hugetlbfs() {
505 assert!(MmapRegion::new(0).is_err());
506
507 let size = 4096;
508
509 let r = MmapRegion::new(size).unwrap();
510 assert_eq!(r.size(), size);
511 assert!(r.file_offset().is_none());
512 assert_eq!(r.prot(), libc::PROT_READ | libc::PROT_WRITE);
513 assert_eq!(
514 r.flags(),
515 libc::MAP_ANONYMOUS | libc::MAP_NORESERVE | libc::MAP_PRIVATE
516 );
517 assert_eq!(r.is_hugetlbfs(), None);
518
519 let mut r = MmapRegion::new(size).unwrap();
520 r.set_hugetlbfs(false);
521 assert_eq!(r.size(), size);
522 assert!(r.file_offset().is_none());
523 assert_eq!(r.prot(), libc::PROT_READ | libc::PROT_WRITE);
524 assert_eq!(
525 r.flags(),
526 libc::MAP_ANONYMOUS | libc::MAP_NORESERVE | libc::MAP_PRIVATE
527 );
528 assert_eq!(r.is_hugetlbfs(), Some(false));
529
530 let mut r = MmapRegion::new(size).unwrap();
531 r.set_hugetlbfs(true);
532 assert_eq!(r.size(), size);
533 assert!(r.file_offset().is_none());
534 assert_eq!(r.prot(), libc::PROT_READ | libc::PROT_WRITE);
535 assert_eq!(
536 r.flags(),
537 libc::MAP_ANONYMOUS | libc::MAP_NORESERVE | libc::MAP_PRIVATE
538 );
539 assert_eq!(r.is_hugetlbfs(), Some(true));
540 }
541
542 #[test]
543 fn test_mmap_region_from_file() {
544 let mut f = TempFile::new().unwrap().into_file();
545 let offset: usize = 0;
546 let buf1 = [1u8, 2, 3, 4, 5];
547
548 f.write_all(buf1.as_ref()).unwrap();
549 let r = MmapRegion::from_file(FileOffset::new(f, offset as u64), buf1.len()).unwrap();
550
551 assert_eq!(r.size(), buf1.len() - offset);
552 assert_eq!(r.file_offset().unwrap().start(), offset as u64);
553 assert_eq!(r.prot(), libc::PROT_READ | libc::PROT_WRITE);
554 assert_eq!(r.flags(), libc::MAP_NORESERVE | libc::MAP_SHARED);
555
556 let buf2 = unsafe { slice::from_raw_parts(r.as_ptr(), buf1.len() - offset) };
557 assert_eq!(&buf1[offset..], buf2);
558 }
559
560 #[test]
561 fn test_mmap_region_build() {
562 let a = Arc::new(TempFile::new().unwrap().into_file());
563
564 let prot = libc::PROT_READ | libc::PROT_WRITE;
565 let flags = libc::MAP_NORESERVE | libc::MAP_PRIVATE;
566 let offset = 4096;
567 let size = 1000;
568
569 let r = MmapRegion::build(
571 Some(FileOffset::from_arc(a.clone(), std::u64::MAX)),
572 size,
573 prot,
574 flags,
575 );
576 assert_eq!(format!("{:?}", r.unwrap_err()), "InvalidOffsetLength");
577
578 let r = MmapRegion::build(
580 Some(FileOffset::from_arc(a.clone(), offset)),
581 size,
582 prot,
583 flags,
584 );
585 assert_eq!(format!("{:?}", r.unwrap_err()), "MappingPastEof");
586
587 let r = MmapRegion::build(
589 Some(FileOffset::from_arc(a.clone(), offset)),
590 size,
591 prot,
592 flags | libc::MAP_FIXED,
593 );
594 assert_eq!(format!("{:?}", r.unwrap_err()), "MapFixed");
595
596 assert_eq!(unsafe { libc::ftruncate(a.as_raw_fd(), 1024 * 10) }, 0);
598
599 let r = MmapRegion::build(
601 Some(FileOffset::from_arc(a.clone(), offset - 1)),
602 size,
603 prot,
604 flags,
605 );
606 assert_eq!(r.unwrap_err().raw_os_error(), libc::EINVAL);
607
608 let r =
610 MmapRegion::build(Some(FileOffset::from_arc(a, offset)), size, prot, flags).unwrap();
611
612 assert_eq!(r.size(), size);
613 assert_eq!(r.file_offset().unwrap().start(), offset as u64);
614 assert_eq!(r.prot(), libc::PROT_READ | libc::PROT_WRITE);
615 assert_eq!(r.flags(), libc::MAP_NORESERVE | libc::MAP_PRIVATE);
616 assert!(r.owned());
617
618 let region_size = 0x10_0000;
619 let bitmap = AtomicBitmap::new(region_size, 0x1000);
620 let builder = MmapRegionBuilder::new_with_bitmap(region_size, bitmap)
621 .with_hugetlbfs(true)
622 .with_mmap_prot(libc::PROT_READ | libc::PROT_WRITE);
623 assert_eq!(builder.size, region_size);
624 assert_eq!(builder.hugetlbfs, Some(true));
625 assert_eq!(builder.prot, libc::PROT_READ | libc::PROT_WRITE);
626
627 crate::bitmap::tests::test_volatile_memory(&(builder.build().unwrap()));
628 }
629
630 #[test]
631 fn test_mmap_region_build_raw() {
632 let addr = 0;
633 let size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize };
634 let prot = libc::PROT_READ | libc::PROT_WRITE;
635 let flags = libc::MAP_NORESERVE | libc::MAP_PRIVATE;
636
637 let r = unsafe { MmapRegion::build_raw((addr + 1) as *mut u8, size, prot, flags) };
638 assert_eq!(format!("{:?}", r.unwrap_err()), "InvalidPointer");
639
640 let r = unsafe { MmapRegion::build_raw(addr as *mut u8, size, prot, flags).unwrap() };
641
642 assert_eq!(r.size(), size);
643 assert_eq!(r.prot(), libc::PROT_READ | libc::PROT_WRITE);
644 assert_eq!(r.flags(), libc::MAP_NORESERVE | libc::MAP_PRIVATE);
645 assert!(!r.owned());
646 }
647
648 #[test]
649 fn test_mmap_region_fds_overlap() {
650 let a = Arc::new(TempFile::new().unwrap().into_file());
651 assert_eq!(unsafe { libc::ftruncate(a.as_raw_fd(), 1024 * 10) }, 0);
652
653 let r1 = MmapRegion::from_file(FileOffset::from_arc(a.clone(), 0), 4096).unwrap();
654 let r2 = MmapRegion::from_file(FileOffset::from_arc(a.clone(), 4096), 4096).unwrap();
655 assert!(!r1.fds_overlap(&r2));
656
657 let r1 = MmapRegion::from_file(FileOffset::from_arc(a.clone(), 0), 5000).unwrap();
658 assert!(r1.fds_overlap(&r2));
659
660 let r2 = MmapRegion::from_file(FileOffset::from_arc(a, 0), 1000).unwrap();
661 assert!(r1.fds_overlap(&r2));
662
663 let new_file = TempFile::new().unwrap().into_file();
665 assert_eq!(
667 unsafe { libc::ftruncate(new_file.as_raw_fd(), 1024 * 10) },
668 0
669 );
670 let r2 = MmapRegion::from_file(FileOffset::new(new_file, 0), 5000).unwrap();
671 assert!(!r1.fds_overlap(&r2));
672
673 let r2 = MmapRegion::new(5000).unwrap();
675 assert!(!r1.fds_overlap(&r2));
676 }
677
678 #[test]
679 fn test_dirty_tracking() {
680 let m = crate::MmapRegion::<AtomicBitmap>::new(0x1_0000).unwrap();
683 crate::bitmap::tests::test_volatile_memory(&m);
684 }
685}