xref: /DragonOS/kernel/src/driver/virtio/transport_pci.rs (revision 86ee1395de7c614865236ee15071c3603b794e44)
1 //! PCI transport for VirtIO.
2 
3 use crate::driver::base::device::DeviceId;
4 use crate::driver::pci::pci::{
5     BusDeviceFunction, PciDeviceStructure, PciDeviceStructureGeneralDevice, PciError,
6     PciStandardDeviceBar, PCI_CAP_ID_VNDR,
7 };
8 
9 use crate::driver::pci::pci_irq::{IrqCommonMsg, IrqSpecificMsg, PciInterrupt, PciIrqMsg, IRQ};
10 use crate::driver::pci::root::pci_root_0;
11 
12 use crate::exception::IrqNumber;
13 
14 use crate::libs::volatile::{
15     volread, volwrite, ReadOnly, Volatile, VolatileReadable, VolatileWritable, WriteOnly,
16 };
17 use crate::mm::VirtAddr;
18 
19 use alloc::string::ToString;
20 use alloc::sync::Arc;
21 use core::{
22     fmt::{self, Display, Formatter},
23     mem::{align_of, size_of},
24     ptr::{self, addr_of_mut, NonNull},
25 };
26 use virtio_drivers::{
27     transport::{DeviceStatus, DeviceType, Transport},
28     Error, Hal, PhysAddr,
29 };
30 
31 use super::irq::DefaultVirtioIrqHandler;
32 use super::VIRTIO_VENDOR_ID;
33 
34 /// The offset to add to a VirtIO device ID to get the corresponding PCI device ID.
35 /// PCI Virtio设备的DEVICE_ID 的offset
36 const PCI_DEVICE_ID_OFFSET: u16 = 0x1040;
37 /// PCI Virtio 设备的DEVICE_ID及其对应的设备类型
38 const TRANSITIONAL_NETWORK: u16 = 0x1000;
39 const TRANSITIONAL_BLOCK: u16 = 0x1001;
40 const TRANSITIONAL_MEMORY_BALLOONING: u16 = 0x1002;
41 const TRANSITIONAL_CONSOLE: u16 = 0x1003;
42 const TRANSITIONAL_SCSI_HOST: u16 = 0x1004;
43 const TRANSITIONAL_ENTROPY_SOURCE: u16 = 0x1005;
44 const TRANSITIONAL_9P_TRANSPORT: u16 = 0x1009;
45 
46 /// The offset of the bar field within `virtio_pci_cap`.
47 const CAP_BAR_OFFSET: u8 = 4;
48 /// The offset of the offset field with `virtio_pci_cap`.
49 const CAP_BAR_OFFSET_OFFSET: u8 = 8;
50 /// The offset of the `length` field within `virtio_pci_cap`.
51 const CAP_LENGTH_OFFSET: u8 = 12;
52 /// The offset of the`notify_off_multiplier` field within `virtio_pci_notify_cap`.
53 const CAP_NOTIFY_OFF_MULTIPLIER_OFFSET: u8 = 16;
54 
55 /// Common configuration.
56 const VIRTIO_PCI_CAP_COMMON_CFG: u8 = 1;
57 /// Notifications.
58 const VIRTIO_PCI_CAP_NOTIFY_CFG: u8 = 2;
59 /// ISR Status.
60 const VIRTIO_PCI_CAP_ISR_CFG: u8 = 3;
61 /// Device specific configuration.
62 const VIRTIO_PCI_CAP_DEVICE_CFG: u8 = 4;
63 
64 /// Virtio设备接收中断的设备号
65 const VIRTIO_RECV_VECTOR: IrqNumber = IrqNumber::new(56);
66 /// Virtio设备接收中断的设备号的表项号
67 const VIRTIO_RECV_VECTOR_INDEX: u16 = 0;
68 // 接收的queue号
69 const QUEUE_RECEIVE: u16 = 0;
70 ///@brief device id 转换为设备类型
71 ///@param pci_device_id,device_id
72 ///@return DeviceType 对应的设备类型
73 fn device_type(pci_device_id: u16) -> DeviceType {
74     match pci_device_id {
75         TRANSITIONAL_NETWORK => DeviceType::Network,
76         TRANSITIONAL_BLOCK => DeviceType::Block,
77         TRANSITIONAL_MEMORY_BALLOONING => DeviceType::MemoryBalloon,
78         TRANSITIONAL_CONSOLE => DeviceType::Console,
79         TRANSITIONAL_SCSI_HOST => DeviceType::ScsiHost,
80         TRANSITIONAL_ENTROPY_SOURCE => DeviceType::EntropySource,
81         TRANSITIONAL_9P_TRANSPORT => DeviceType::_9P,
82         id if id >= PCI_DEVICE_ID_OFFSET => DeviceType::from(id - PCI_DEVICE_ID_OFFSET),
83         _ => DeviceType::Invalid,
84     }
85 }
86 
87 /// PCI transport for VirtIO.
88 ///
89 /// Ref: 4.1 Virtio Over PCI Bus
90 #[allow(dead_code)]
91 #[derive(Debug, Clone)]
92 pub struct PciTransport {
93     device_type: DeviceType,
94     /// The bus, device and function identifier for the VirtIO device.
95     _bus_device_function: BusDeviceFunction,
96     /// The common configuration structure within some BAR.
97     common_cfg: NonNull<CommonCfg>,
98     /// The start of the queue notification region within some BAR.
99     notify_region: NonNull<[WriteOnly<u16>]>,
100     notify_off_multiplier: u32,
101     /// The ISR status register within some BAR.
102     isr_status: NonNull<Volatile<u8>>,
103     /// The VirtIO device-specific configuration within some BAR.
104     config_space: Option<NonNull<[u32]>>,
105     irq: IrqNumber,
106     dev_id: Arc<DeviceId>,
107 }
108 
109 impl PciTransport {
110     /// Construct a new PCI VirtIO device driver for the given device function on the given PCI
111     /// root controller.
112     ///
113     /// ## 参数
114     ///
115     /// - `device` - The PCI device structure for the VirtIO device.
116     /// - `irq_handler` - An optional handler for the device's interrupt. If `None`, a default
117     ///     handler `DefaultVirtioIrqHandler` will be used.
118     #[allow(clippy::extra_unused_type_parameters)]
119     pub fn new<H: Hal>(
120         device: &mut PciDeviceStructureGeneralDevice,
121         dev_id: Arc<DeviceId>,
122     ) -> Result<Self, VirtioPciError> {
123         let irq = VIRTIO_RECV_VECTOR;
124         let header = &device.common_header;
125         let bus_device_function = header.bus_device_function;
126         if header.vendor_id != VIRTIO_VENDOR_ID {
127             return Err(VirtioPciError::InvalidVendorId(header.vendor_id));
128         }
129         let device_type = device_type(header.device_id);
130         // Find the PCI capabilities we need.
131         let mut common_cfg: Option<VirtioCapabilityInfo> = None;
132         let mut notify_cfg: Option<VirtioCapabilityInfo> = None;
133         let mut notify_off_multiplier = 0;
134         let mut isr_cfg = None;
135         let mut device_cfg = None;
136         device.bar_ioremap().unwrap()?;
137         device.enable_master();
138         let standard_device = device.as_standard_device_mut().unwrap();
139         // 目前缺少对PCI设备中断号的统一管理,所以这里需要指定一个中断号。不能与其他中断重复
140         let irq_vector = standard_device.irq_vector_mut().unwrap();
141         irq_vector.push(irq);
142         standard_device
143             .irq_init(IRQ::PCI_IRQ_MSIX)
144             .expect("IRQ init failed");
145         // 中断相关信息
146         let msg = PciIrqMsg {
147             irq_common_message: IrqCommonMsg::init_from(
148                 0,
149                 "Virtio_IRQ".to_string(),
150                 &DefaultVirtioIrqHandler,
151                 dev_id.clone(),
152             ),
153             irq_specific_message: IrqSpecificMsg::msi_default(),
154         };
155         standard_device.irq_install(msg)?;
156         standard_device.irq_enable(true)?;
157         //device_capability为迭代器,遍历其相当于遍历所有的cap空间
158         for capability in device.capabilities().unwrap() {
159             if capability.id != PCI_CAP_ID_VNDR {
160                 continue;
161             }
162             let cap_len = capability.private_header as u8;
163             let cfg_type = (capability.private_header >> 8) as u8;
164             if cap_len < 16 {
165                 continue;
166             }
167             let struct_info = VirtioCapabilityInfo {
168                 bar: pci_root_0().read_config(
169                     bus_device_function,
170                     (capability.offset + CAP_BAR_OFFSET).into(),
171                 ) as u8,
172                 offset: pci_root_0().read_config(
173                     bus_device_function,
174                     (capability.offset + CAP_BAR_OFFSET_OFFSET).into(),
175                 ),
176                 length: pci_root_0().read_config(
177                     bus_device_function,
178                     (capability.offset + CAP_LENGTH_OFFSET).into(),
179                 ),
180             };
181 
182             match cfg_type {
183                 VIRTIO_PCI_CAP_COMMON_CFG if common_cfg.is_none() => {
184                     common_cfg = Some(struct_info);
185                 }
186                 VIRTIO_PCI_CAP_NOTIFY_CFG if cap_len >= 20 && notify_cfg.is_none() => {
187                     notify_cfg = Some(struct_info);
188                     notify_off_multiplier = pci_root_0().read_config(
189                         bus_device_function,
190                         (capability.offset + CAP_NOTIFY_OFF_MULTIPLIER_OFFSET).into(),
191                     );
192                 }
193                 VIRTIO_PCI_CAP_ISR_CFG if isr_cfg.is_none() => {
194                     isr_cfg = Some(struct_info);
195                 }
196                 VIRTIO_PCI_CAP_DEVICE_CFG if device_cfg.is_none() => {
197                     device_cfg = Some(struct_info);
198                 }
199                 _ => {}
200             }
201         }
202 
203         let common_cfg = get_bar_region::<_>(
204             &device.standard_device_bar,
205             &common_cfg.ok_or(VirtioPciError::MissingCommonConfig)?,
206         )?;
207 
208         let notify_cfg = notify_cfg.ok_or(VirtioPciError::MissingNotifyConfig)?;
209         if notify_off_multiplier % 2 != 0 {
210             return Err(VirtioPciError::InvalidNotifyOffMultiplier(
211                 notify_off_multiplier,
212             ));
213         }
214         //debug!("notify.offset={},notify.length={}",notify_cfg.offset,notify_cfg.length);
215         let notify_region = get_bar_region_slice::<_>(&device.standard_device_bar, &notify_cfg)?;
216         let isr_status = get_bar_region::<_>(
217             &device.standard_device_bar,
218             &isr_cfg.ok_or(VirtioPciError::MissingIsrConfig)?,
219         )?;
220         let config_space = if let Some(device_cfg) = device_cfg {
221             Some(get_bar_region_slice::<_>(
222                 &device.standard_device_bar,
223                 &device_cfg,
224             )?)
225         } else {
226             None
227         };
228         Ok(Self {
229             device_type,
230             _bus_device_function: bus_device_function,
231             common_cfg,
232             notify_region,
233             notify_off_multiplier,
234             isr_status,
235             config_space,
236             irq,
237             dev_id,
238         })
239     }
240 }
241 
242 impl Transport for PciTransport {
243     fn device_type(&self) -> DeviceType {
244         self.device_type
245     }
246 
247     fn read_device_features(&mut self) -> u64 {
248         // Safe because the common config pointer is valid and we checked in get_bar_region that it
249         // was aligned.
250         unsafe {
251             volwrite!(self.common_cfg, device_feature_select, 0);
252             let mut device_features_bits = volread!(self.common_cfg, device_feature) as u64;
253             volwrite!(self.common_cfg, device_feature_select, 1);
254             device_features_bits |= (volread!(self.common_cfg, device_feature) as u64) << 32;
255             device_features_bits
256         }
257     }
258 
259     fn write_driver_features(&mut self, driver_features: u64) {
260         // Safe because the common config pointer is valid and we checked in get_bar_region that it
261         // was aligned.
262         unsafe {
263             volwrite!(self.common_cfg, driver_feature_select, 0);
264             volwrite!(self.common_cfg, driver_feature, driver_features as u32);
265             volwrite!(self.common_cfg, driver_feature_select, 1);
266             volwrite!(
267                 self.common_cfg,
268                 driver_feature,
269                 (driver_features >> 32) as u32
270             );
271         }
272     }
273 
274     fn max_queue_size(&mut self, queue: u16) -> u32 {
275         unsafe {
276             volwrite!(self.common_cfg, queue_select, queue);
277             volread!(self.common_cfg, queue_size).into()
278         }
279     }
280 
281     fn notify(&mut self, queue: u16) {
282         // Safe because the common config and notify region pointers are valid and we checked in
283         // get_bar_region that they were aligned.
284         unsafe {
285             volwrite!(self.common_cfg, queue_select, queue);
286             // TODO: Consider caching this somewhere (per queue).
287             let queue_notify_off = volread!(self.common_cfg, queue_notify_off);
288 
289             let offset_bytes = usize::from(queue_notify_off) * self.notify_off_multiplier as usize;
290             let index = offset_bytes / size_of::<u16>();
291             addr_of_mut!((*self.notify_region.as_ptr())[index]).vwrite(queue);
292         }
293     }
294 
295     fn set_status(&mut self, status: DeviceStatus) {
296         // Safe because the common config pointer is valid and we checked in get_bar_region that it
297         // was aligned.
298         unsafe {
299             volwrite!(self.common_cfg, device_status, status.bits() as u8);
300         }
301     }
302 
303     fn get_status(&self) -> DeviceStatus {
304         // Safe because the common config pointer is valid and we checked in get_bar_region that it
305         // was aligned.
306         unsafe { DeviceStatus::from_bits_truncate(volread!(self.common_cfg, device_status).into()) }
307     }
308 
309     fn set_guest_page_size(&mut self, _guest_page_size: u32) {
310         // No-op, the PCI transport doesn't care.
311     }
312     fn requires_legacy_layout(&self) -> bool {
313         false
314     }
315     fn queue_set(
316         &mut self,
317         queue: u16,
318         size: u32,
319         descriptors: PhysAddr,
320         driver_area: PhysAddr,
321         device_area: PhysAddr,
322     ) {
323         // Safe because the common config pointer is valid and we checked in get_bar_region that it
324         // was aligned.
325         unsafe {
326             volwrite!(self.common_cfg, queue_select, queue);
327             volwrite!(self.common_cfg, queue_size, size as u16);
328             volwrite!(self.common_cfg, queue_desc, descriptors as u64);
329             volwrite!(self.common_cfg, queue_driver, driver_area as u64);
330             volwrite!(self.common_cfg, queue_device, device_area as u64);
331             // 这里设置队列中断对应的中断项
332             if queue == QUEUE_RECEIVE {
333                 volwrite!(self.common_cfg, queue_msix_vector, VIRTIO_RECV_VECTOR_INDEX);
334                 let vector = volread!(self.common_cfg, queue_msix_vector);
335                 if vector != VIRTIO_RECV_VECTOR_INDEX {
336                     panic!("Vector set failed");
337                 }
338             }
339             volwrite!(self.common_cfg, queue_enable, 1);
340         }
341     }
342 
343     fn queue_unset(&mut self, queue: u16) {
344         // Safe because the common config pointer is valid and we checked in get_bar_region that it
345         // was aligned.
346         unsafe {
347             volwrite!(self.common_cfg, queue_select, queue);
348             volwrite!(self.common_cfg, queue_size, 0);
349             volwrite!(self.common_cfg, queue_desc, 0);
350             volwrite!(self.common_cfg, queue_driver, 0);
351             volwrite!(self.common_cfg, queue_device, 0);
352         }
353     }
354 
355     fn queue_used(&mut self, queue: u16) -> bool {
356         // Safe because the common config pointer is valid and we checked in get_bar_region that it
357         // was aligned.
358         unsafe {
359             volwrite!(self.common_cfg, queue_select, queue);
360             volread!(self.common_cfg, queue_enable) == 1
361         }
362     }
363 
364     fn ack_interrupt(&mut self) -> bool {
365         // Safe because the common config pointer is valid and we checked in get_bar_region that it
366         // was aligned.
367         // Reading the ISR status resets it to 0 and causes the device to de-assert the interrupt.
368         let isr_status = unsafe { self.isr_status.as_ptr().vread() };
369         // TODO: Distinguish between queue interrupt and device configuration interrupt.
370         isr_status & 0x3 != 0
371     }
372 
373     fn config_space<T>(&self) -> Result<NonNull<T>, Error> {
374         if let Some(config_space) = self.config_space {
375             if size_of::<T>() > config_space.len() * size_of::<u32>() {
376                 Err(Error::ConfigSpaceTooSmall)
377             } else if align_of::<T>() > 4 {
378                 // Panic as this should only happen if the driver is written incorrectly.
379                 panic!(
380                     "Driver expected config space alignment of {} bytes, but VirtIO only guarantees 4 byte alignment.",
381                     align_of::<T>()
382                 );
383             } else {
384                 // TODO: Use NonNull::as_non_null_ptr once it is stable.
385                 let config_space_ptr = NonNull::new(config_space.as_ptr() as *mut u32).unwrap();
386                 Ok(config_space_ptr.cast())
387             }
388         } else {
389             Err(Error::ConfigSpaceMissing)
390         }
391     }
392 }
393 
394 impl Drop for PciTransport {
395     fn drop(&mut self) {
396         // Reset the device when the transport is dropped.
397         self.set_status(DeviceStatus::empty());
398 
399         // todo: 调用pci的中断释放函数,并且在virtio_irq_manager里面删除对应的设备的中断
400     }
401 }
402 
403 #[repr(C)]
404 struct CommonCfg {
405     device_feature_select: Volatile<u32>,
406     device_feature: ReadOnly<u32>,
407     driver_feature_select: Volatile<u32>,
408     driver_feature: Volatile<u32>,
409     msix_config: Volatile<u16>,
410     num_queues: ReadOnly<u16>,
411     device_status: Volatile<u8>,
412     config_generation: ReadOnly<u8>,
413     queue_select: Volatile<u16>,
414     queue_size: Volatile<u16>,
415     queue_msix_vector: Volatile<u16>,
416     queue_enable: Volatile<u16>,
417     queue_notify_off: Volatile<u16>,
418     queue_desc: Volatile<u64>,
419     queue_driver: Volatile<u64>,
420     queue_device: Volatile<u64>,
421 }
422 
423 /// Information about a VirtIO structure within some BAR, as provided by a `virtio_pci_cap`.
424 /// cfg空间在哪个bar的多少偏移处,长度多少
425 #[derive(Clone, Debug, Eq, PartialEq)]
426 struct VirtioCapabilityInfo {
427     /// The bar in which the structure can be found.
428     bar: u8,
429     /// The offset within the bar.
430     offset: u32,
431     /// The length in bytes of the structure within the bar.
432     length: u32,
433 }
434 
435 /// An error encountered initialising a VirtIO PCI transport.
436 /// VirtIO PCI transport 初始化时的错误
437 #[derive(Clone, Debug, Eq, PartialEq)]
438 pub enum VirtioPciError {
439     /// PCI device vender ID was not the VirtIO vendor ID.
440     InvalidVendorId(u16),
441     /// No valid `VIRTIO_PCI_CAP_COMMON_CFG` capability was found.
442     MissingCommonConfig,
443     /// No valid `VIRTIO_PCI_CAP_NOTIFY_CFG` capability was found.
444     MissingNotifyConfig,
445     /// `VIRTIO_PCI_CAP_NOTIFY_CFG` capability has a `notify_off_multiplier` that is not a multiple
446     /// of 2.
447     InvalidNotifyOffMultiplier(u32),
448     /// No valid `VIRTIO_PCI_CAP_ISR_CFG` capability was found.
449     MissingIsrConfig,
450     /// An IO BAR was provided rather than a memory BAR.
451     UnexpectedBarType,
452     /// A BAR which we need was not allocated an address.
453     BarNotAllocated(u8),
454     /// The offset for some capability was greater than the length of the BAR.
455     BarOffsetOutOfRange,
456     /// The virtual address was not aligned as expected.
457     Misaligned {
458         /// The virtual address in question.
459         vaddr: VirtAddr,
460         /// The expected alignment in bytes.
461         alignment: usize,
462     },
463     ///获取虚拟地址失败
464     BarGetVaddrFailed,
465     /// A generic PCI error,
466     Pci(PciError),
467 }
468 
469 impl Display for VirtioPciError {
470     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
471         match self {
472             Self::InvalidVendorId(vendor_id) => write!(
473                 f,
474                 "PCI device vender ID {:#06x} was not the VirtIO vendor ID {:#06x}.",
475                 vendor_id, VIRTIO_VENDOR_ID
476             ),
477             Self::MissingCommonConfig => write!(
478                 f,
479                 "No valid `VIRTIO_PCI_CAP_COMMON_CFG` capability was found."
480             ),
481             Self::MissingNotifyConfig => write!(
482                 f,
483                 "No valid `VIRTIO_PCI_CAP_NOTIFY_CFG` capability was found."
484             ),
485             Self::InvalidNotifyOffMultiplier(notify_off_multiplier) => {
486                 write!(
487                     f,
488                     "`VIRTIO_PCI_CAP_NOTIFY_CFG` capability has a `notify_off_multiplier` that is not a multiple of 2: {}",
489                     notify_off_multiplier
490                 )
491             }
492             Self::MissingIsrConfig => {
493                 write!(f, "No valid `VIRTIO_PCI_CAP_ISR_CFG` capability was found.")
494             }
495             Self::UnexpectedBarType => write!(f, "Unexpected BAR (expected memory BAR)."),
496             Self::BarNotAllocated(bar_index) => write!(f, "Bar {} not allocated.", bar_index),
497             Self::BarOffsetOutOfRange => write!(f, "Capability offset greater than BAR length."),
498             Self::Misaligned { vaddr, alignment } => write!(
499                 f,
500                 "Virtual address {:?} was not aligned to a {} byte boundary as expected.",
501                 vaddr, alignment
502             ),
503             Self::BarGetVaddrFailed => write!(f, "Get bar virtaddress failed"),
504             Self::Pci(pci_error) => pci_error.fmt(f),
505         }
506     }
507 }
508 
509 /// PCI error到VirtioPciError的转换,层层上报
510 impl From<PciError> for VirtioPciError {
511     fn from(error: PciError) -> Self {
512         Self::Pci(error)
513     }
514 }
515 
516 /// @brief 获取虚拟地址并将其转化为对应类型的指针
517 /// @param device_bar 存储bar信息的结构体 struct_info 存储cfg空间的位置信息
518 /// @return Result<NonNull<T>, VirtioPciError> 成功则返回对应类型的指针,失败则返回Error
519 fn get_bar_region<T>(
520     device_bar: &PciStandardDeviceBar,
521     struct_info: &VirtioCapabilityInfo,
522 ) -> Result<NonNull<T>, VirtioPciError> {
523     let bar_info = device_bar.get_bar(struct_info.bar)?;
524     let (bar_address, bar_size) = bar_info
525         .memory_address_size()
526         .ok_or(VirtioPciError::UnexpectedBarType)?;
527     if bar_address == 0 {
528         return Err(VirtioPciError::BarNotAllocated(struct_info.bar));
529     }
530     if struct_info.offset + struct_info.length > bar_size
531         || size_of::<T>() > struct_info.length as usize
532     {
533         return Err(VirtioPciError::BarOffsetOutOfRange);
534     }
535     //debug!("Chossed bar ={},used={}",struct_info.bar,struct_info.offset + struct_info.length);
536     let vaddr = (bar_info
537         .virtual_address()
538         .ok_or(VirtioPciError::BarGetVaddrFailed)?)
539         + struct_info.offset as usize;
540     if vaddr.data() % align_of::<T>() != 0 {
541         return Err(VirtioPciError::Misaligned {
542             vaddr,
543             alignment: align_of::<T>(),
544         });
545     }
546     let vaddr = NonNull::new(vaddr.data() as *mut u8).unwrap();
547     Ok(vaddr.cast())
548 }
549 
550 /// @brief 获取虚拟地址并将其转化为对应类型的切片的指针
551 /// @param device_bar 存储bar信息的结构体 struct_info 存储cfg空间的位置信息切片的指针
552 /// @return Result<NonNull<[T]>, VirtioPciError> 成功则返回对应类型的指针切片,失败则返回Error
553 fn get_bar_region_slice<T>(
554     device_bar: &PciStandardDeviceBar,
555     struct_info: &VirtioCapabilityInfo,
556 ) -> Result<NonNull<[T]>, VirtioPciError> {
557     let ptr = get_bar_region::<T>(device_bar, struct_info)?;
558     // let raw_slice =
559     //     ptr::slice_from_raw_parts_mut(ptr.as_ptr(), struct_info.length as usize / size_of::<T>());
560     Ok(nonnull_slice_from_raw_parts(
561         ptr,
562         struct_info.length as usize / size_of::<T>(),
563     ))
564 }
565 
566 fn nonnull_slice_from_raw_parts<T>(data: NonNull<T>, len: usize) -> NonNull<[T]> {
567     NonNull::new(ptr::slice_from_raw_parts_mut(data.as_ptr(), len)).unwrap()
568 }
569