/**************************************************************************** * drivers/virtio/virtio-pci-modern.c * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. The * ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. * ****************************************************************************/ /**************************************************************************** * Included Files ****************************************************************************/ #include #include #include #include #include "virtio-pci.h" /**************************************************************************** * Pre-processor Definitions ****************************************************************************/ /* Common configuration */ #define VIRTIO_PCI_CAP_COMMON_CFG 1 /* Notifications */ #define VIRTIO_PCI_CAP_NOTIFY_CFG 2 /* ISR Status */ #define VIRTIO_PCI_CAP_ISR_CFG 3 /* Device specific configuration */ #define VIRTIO_PCI_CAP_DEVICE_CFG 4 /* PCI configuration access */ #define VIRTIO_PCI_CAP_PCI_CFG 5 /* Shared memory region */ #define VIRTIO_PCI_CAP_SHARED_MEMORY_CFG 8 /* Vendor-specific data */ #define VIRTIO_PCI_CAP_VENDOR_CFG 9 /**************************************************************************** * Private Types ****************************************************************************/ begin_packed_struct struct virtio_pci_cap_s { uint8_t cap_vndr; /* Generic PCI field: PCI_CAP_ID_VNDR */ uint8_t cap_next; /* Generic PCI field: next ptr. */ uint8_t cap_len; /* Generic PCI field: capability length */ uint8_t cfg_type; /* Identifies the structure. */ uint8_t bar; /* Where to find it. */ uint8_t id; /* Multiple capabilities of the same type */ uint8_t padding[2]; /* Pad to full dword. */ uint32_t offset; /* Offset within bar. */ uint32_t length; /* Length of the structure, in bytes. */ } end_packed_struct; begin_packed_struct struct virtio_pci_cap64_s { struct virtio_pci_cap_s cap; uint32_t offset_hi; uint32_t length_hi; } end_packed_struct; /* VIRTIO_PCI_CAP_COMMON_CFG */ begin_packed_struct struct virtio_pci_common_cfg_s { /* About the whole device. */ uint32_t device_feature_select; /* read-write */ uint32_t device_feature; /* read-only for driver */ uint32_t driver_feature_select; /* read-write */ uint32_t driver_feature; /* read-write */ uint16_t config_msix_vector; /* read-write */ uint16_t num_queues; /* read-only for driver */ uint8_t device_status; /* read-write */ uint8_t config_generation; /* read-only for driver */ /* About a specific virtqueue. */ uint16_t queue_select; /* read-write */ uint16_t queue_size; /* read-write */ uint16_t queue_msix_vector; /* read-write */ uint16_t queue_enable; /* read-write */ uint16_t queue_notify_off; /* read-only for driver */ uint64_t queue_desc; /* read-write */ uint64_t queue_avail; /* read-write */ uint64_t queue_used; /* read-write */ uint16_t queue_notify_data; /* read-only for driver */ uint16_t queue_reset; /* read-write */ } end_packed_struct; /* VIRTIO_PCI_CAP_NOTIFY_CFG */ begin_packed_struct struct virtio_pci_notify_cap_s { struct virtio_pci_cap_s cap; /* Multiplier for queue_notify_off. */ uint32_t notify_off_multiplier; } end_packed_struct; /* VIRTIO_PCI_CAP_VENDOR_CFG */ begin_packed_struct struct virtio_pci_vndr_data_s { uint8_t cap_vndr; /* Generic PCI field: PCI_CAP_ID_VNDR */ uint8_t cap_next; /* Generic PCI field: next ptr. */ uint8_t cao_len; /* Generic PCI field: capability length */ uint8_t cfg_type; /* Identifies the structure. */ uint16_t vendor_id; /* Identifies the vendor-specific format. */ /* For Vendor Definition * Pads structure to a multiple of 4 bytes * Reads must not have side effects */ } end_packed_struct; /* VIRTIO_PCI_CAP_PCI_CFG */ begin_packed_struct struct virtio_pci_cfg_cap_s { struct virtio_pci_cap_s cap; uint8_t pci_cfg_data[4]; /* Data for BAR access. */ } end_packed_struct; /**************************************************************************** * Private Function Prototypes ****************************************************************************/ /* Helper functions */ static uint16_t virtio_pci_modern_get_queue_len(FAR struct virtio_pci_device_s *vpdev, int idx); static int virtio_pci_modern_config_vector(FAR struct virtio_pci_device_s *vpdev, bool enable); static int virtio_pci_modern_create_virtqueue(FAR struct virtio_pci_device_s *vpdev, FAR struct virtqueue *vq); static void virtio_pci_modern_delete_virtqueue(FAR struct virtio_device *vdev, int index); static void virtio_pci_modern_set_status(FAR struct virtio_device *vdev, uint8_t status); static uint8_t virtio_pci_modern_get_status(FAR struct virtio_device *vdev); static void virtio_pci_modern_write_config(FAR struct virtio_device *vdev, uint32_t offset, FAR void *dst, int length); static void virtio_pci_modern_read_config(FAR struct virtio_device *vdev, uint32_t offset, FAR void *dst, int length); static uint32_t virtio_pci_modern_get_features(FAR struct virtio_device *vdev); static void virtio_pci_modern_set_features(FAR struct virtio_device *vdev, uint32_t features); static void virtio_pci_modern_notify(FAR struct virtqueue *vq); /**************************************************************************** * Private Data ****************************************************************************/ static const struct virtio_dispatch g_virtio_pci_dispatch = { virtio_pci_create_virtqueues, /* create_virtqueues */ virtio_pci_delete_virtqueues, /* delete_virtqueues */ virtio_pci_modern_get_status, /* get_status */ virtio_pci_modern_set_status, /* set_status */ virtio_pci_modern_get_features, /* get_features */ virtio_pci_modern_set_features, /* set_features */ virtio_pci_negotiate_features, /* negotiate_features */ virtio_pci_modern_read_config, /* read_config */ virtio_pci_modern_write_config, /* write_config */ virtio_pci_reset_device, /* reset_device */ virtio_pci_modern_notify, /* notify */ }; static const struct virtio_pci_ops_s g_virtio_pci_modern_ops = { virtio_pci_modern_get_queue_len, /* get_queue_len */ virtio_pci_modern_config_vector, /* config_vector */ virtio_pci_modern_create_virtqueue, /* create_virtqueue */ virtio_pci_modern_delete_virtqueue, /* delete_virtqueue */ }; /**************************************************************************** * Private Functions ****************************************************************************/ /**************************************************************************** * Name: virtio_pci_find_capability ****************************************************************************/ static uint8_t virtio_pci_find_capability(FAR struct pci_device_s *dev, uint8_t cfg_type, unsigned int flags) { uint8_t type; uint8_t bar; uint8_t pos; for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0; pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap_s, cfg_type), &type); pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap_s, bar), &bar); /* VirtIO Spec v1.2: The driver MUST ignore any vendor-specific * capability structure which has a reserved cfg_type value. */ if (bar >= PCI_STD_NUM_BARS) { continue; } if (type == cfg_type && pci_resource_len(dev, bar) && (pci_resource_flags(dev, bar) & flags)) { return pos; } } return 0; } /**************************************************************************** * Name: virtio_pci_map_capability ****************************************************************************/ static FAR void * virtio_pci_map_capability(FAR struct virtio_pci_device_s *vpdev, uint8_t pos, size_t *len) { FAR struct pci_device_s *dev = vpdev->dev; FAR void *ptr; uint32_t offset; uint32_t length; uint8_t bar; pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap_s, bar), &bar); pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap_s, offset), &offset); pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap_s, length), &length); if (bar >= PCI_STD_NUM_BARS) { vrterr("Bar error bard=%u\n", bar); return NULL; } ptr = pci_map_bar_region(dev, bar, offset, length); if (ptr == NULL) { vrterr("Unable to map virtio on bar %u\n", bar); return NULL; } if (len) { *len = length; } return ptr; } /**************************************************************************** * Name: virtio_pci_modern_get_queue_len ****************************************************************************/ static uint16_t virtio_pci_modern_get_queue_len(FAR struct virtio_pci_device_s *vpdev, int idx) { FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; uint16_t size; pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_select, idx); pci_read_io_word(vpdev->dev, (uintptr_t)&cfg->queue_size, &size); return size; } /**************************************************************************** * Name: virtio_pci_modern_create_virtqueue ****************************************************************************/ static int virtio_pci_modern_create_virtqueue(FAR struct virtio_pci_device_s *vpdev, FAR struct virtqueue *vq) { FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; FAR struct virtio_vring_info *vrinfo; #if CONFIG_DRIVERS_VIRTIO_PCI_POLLING_PERIOD <= 0 uint16_t msix_vector; #endif uint16_t off; /* Activate the queue */ pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_select, vq->vq_queue_index); pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_size, vq->vq_ring.num); pci_write_io_qword(vpdev->dev, (uintptr_t)&cfg->queue_desc, up_addrenv_va_to_pa(vq->vq_ring.desc)); pci_write_io_qword(vpdev->dev, (uintptr_t)&cfg->queue_avail, up_addrenv_va_to_pa(vq->vq_ring.avail)); pci_write_io_qword(vpdev->dev, (uintptr_t)&cfg->queue_used, up_addrenv_va_to_pa(vq->vq_ring.used)); #if CONFIG_DRIVERS_VIRTIO_PCI_POLLING_PERIOD <= 0 pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_msix_vector, VIRTIO_PCI_INT_VQ); pci_read_io_word(vpdev->dev, (uintptr_t)&cfg->queue_msix_vector, &msix_vector); if (msix_vector == VIRTIO_PCI_MSI_NO_VECTOR) { vrterr("Msix_vector is no vector\n"); return -EBUSY; } #endif /* Enable vq */ pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_enable, 1); pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_select, vq->vq_queue_index); pci_read_io_word(vpdev->dev, (uintptr_t)&cfg->queue_notify_off, &off); if (off * vpdev->notify_off_multiplier + 2 > vpdev->notify_len) { return -EINVAL; } vrinfo = &vpdev->vdev.vrings_info[vq->vq_queue_index]; vrinfo->notifyid = off * vpdev->notify_off_multiplier; return OK; } /**************************************************************************** * Name: virtio_pci_modern_config_vector ****************************************************************************/ static int virtio_pci_modern_config_vector(FAR struct virtio_pci_device_s *vpdev, bool enable) { FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; uint16_t vector = enable ? 0 : VIRTIO_PCI_MSI_NO_VECTOR; uint16_t rvector; pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->config_msix_vector, vector); pci_read_io_word(vpdev->dev, (uintptr_t)&cfg->config_msix_vector, &rvector); if (rvector != vector) { return -EINVAL; } return OK; } /**************************************************************************** * Name: virtio_pci_modern_delete_virtqueue ****************************************************************************/ void virtio_pci_modern_delete_virtqueue(FAR struct virtio_device *vdev, int index) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vdev; FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_select, index); #if CONFIG_DRIVERS_VIRTIO_PCI_POLLING_PERIOD <= 0 pci_write_io_word(vpdev->dev, (uintptr_t)&cfg->queue_msix_vector, VIRTIO_PCI_MSI_NO_VECTOR); #endif } /**************************************************************************** * Name: virtio_pci_modern_set_status ****************************************************************************/ static void virtio_pci_modern_set_status(FAR struct virtio_device *vdev, uint8_t status) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vdev; FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; pci_write_io_byte(vpdev->dev, (uintptr_t)&cfg->device_status, status); } /**************************************************************************** * Name: virtio_pci_modern_get_status ****************************************************************************/ static uint8_t virtio_pci_modern_get_status(FAR struct virtio_device *vdev) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vdev; FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; uint8_t status; pci_read_io_byte(vpdev->dev, (uintptr_t)&cfg->device_status, &status); return status; } /**************************************************************************** * Name: virtio_pci_modern_write_config ****************************************************************************/ static void virtio_pci_modern_write_config(FAR struct virtio_device *vdev, uint32_t offset, FAR void *src, int length) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vdev; uint64_t u64data; uint32_t u32data; uint16_t u16data; uint8_t u8data; if (offset + length > vpdev->device_len) { return; } switch (length) { case 1: memcpy(&u8data, src, sizeof(u8data)); pci_write_io_byte(vpdev->dev, (uintptr_t)(vpdev->device + offset), u8data); break; case 2: memcpy(&u16data, src, sizeof(u16data)); pci_write_io_word(vpdev->dev, (uintptr_t)(vpdev->device + offset), u16data); break; case 4: memcpy(&u32data, src, sizeof(u32data)); pci_write_io_dword(vpdev->dev, (uintptr_t)(vpdev->device + offset), u32data); break; case 8: memcpy(&u64data, src, sizeof(u64data)); pci_write_io_qword(vpdev->dev, (uintptr_t)(vpdev->device + offset), u64data); break; default: { FAR char *s = src; int i; for (i = 0; i < length; i++) { pci_write_io_byte(vpdev->dev, (uintptr_t)(vpdev->device + offset + i), s[i]); } } } } /**************************************************************************** * Name: virtio_pci_modern_read_config ****************************************************************************/ static void virtio_pci_modern_read_config(FAR struct virtio_device *vdev, uint32_t offset, FAR void *dst, int length) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vdev; uint64_t u64data; uint32_t u32data; uint16_t u16data; uint8_t u8data; if (offset + length > vpdev->device_len) { return; } switch (length) { case 1: pci_read_io_byte(vpdev->dev, (uintptr_t)(vpdev->device + offset), &u8data); memcpy(dst, &u8data, sizeof(u8data)); break; case 2: pci_read_io_word(vpdev->dev, (uintptr_t)(vpdev->device + offset), &u16data); memcpy(dst, &u16data, sizeof(u16data)); break; case 4: pci_read_io_dword(vpdev->dev, (uintptr_t)(vpdev->device + offset), &u32data); memcpy(dst, &u32data, sizeof(u32data)); break; case 8: pci_read_io_qword(vpdev->dev, (uintptr_t)(vpdev->device + offset), &u64data); memcpy(dst, &u64data, sizeof(u64data)); break; default: { FAR char *d = dst; int i; for (i = 0; i < length; i++) { pci_read_io_byte(vpdev->dev, (uintptr_t)(vpdev->device + offset + i), (uint8_t *)&d[i]); } } } } /**************************************************************************** * Name: virtio_pci_modern_get_features ****************************************************************************/ static uint32_t virtio_pci_modern_get_features(FAR struct virtio_device *vdev) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vdev; FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; uint32_t feature; pci_write_io_dword(vpdev->dev, (uintptr_t)&cfg->device_feature_select, 0); pci_read_io_dword(vpdev->dev, (uintptr_t)&cfg->device_feature, &feature); return feature; } /**************************************************************************** * Name: virtio_pci_modern_set_features ****************************************************************************/ static void virtio_pci_modern_set_features(FAR struct virtio_device *vdev, uint32_t features) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vdev; FAR struct virtio_pci_common_cfg_s *cfg = vpdev->common; pci_write_io_dword(vpdev->dev, (uintptr_t)&cfg->driver_feature_select, 0); pci_write_io_dword(vpdev->dev, (uintptr_t)&cfg->driver_feature, features); pci_write_io_dword(vpdev->dev, (uintptr_t)&cfg->driver_feature_select, 1); pci_write_io_dword(vpdev->dev, (uintptr_t)&cfg->driver_feature, 0); vdev->features = features; } /**************************************************************************** * Name: virtio_pci_modern_notify ****************************************************************************/ static void virtio_pci_modern_notify(FAR struct virtqueue *vq) { FAR struct virtio_pci_device_s *vpdev = (FAR struct virtio_pci_device_s *)vq->vq_dev; FAR struct virtio_vring_info *vrinfo; vrinfo = &vpdev->vdev.vrings_info[vq->vq_queue_index]; pci_write_io_word(vpdev->dev, (uintptr_t)(vpdev->notify + vrinfo->notifyid), vq->vq_queue_index); } /**************************************************************************** * Name: virtio_pci_probe ****************************************************************************/ static int virtio_pci_init_device(FAR struct virtio_pci_device_s *vpdev) { FAR struct virtio_device *vdev = &vpdev->vdev; FAR struct pci_device_s *dev = vpdev->dev; uint8_t common; uint8_t notify; uint8_t device; if (dev->device < 0x1040) { /* Transitional devices: use the PCI subsystem device id as * virtio device id, same as legacy driver always did. */ vdev->id.device = dev->subsystem_device; } else { /* Modern devices: simply use PCI device id, but start from 0x1040. */ vdev->id.device = dev->device - 0x1040; } vdev->id.vendor = dev->subsystem_vendor; /* Find pci capabilities */ common = virtio_pci_find_capability(dev, VIRTIO_PCI_CAP_COMMON_CFG, PCI_RESOURCE_MEM); if (common == 0) { vrtinfo("Leaving for legacy driver\n"); return -ENODEV; } notify = virtio_pci_find_capability(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, PCI_RESOURCE_MEM); if (notify == 0) { vrterr("Missing capabilities %d %d\n", common, notify); return -EINVAL; } device = virtio_pci_find_capability(dev, VIRTIO_PCI_CAP_DEVICE_CFG, PCI_RESOURCE_MEM); /* Map the BAR */ vpdev->common = virtio_pci_map_capability(vpdev, common, &vpdev->common_len); if (vpdev->common == NULL) { return -EINVAL; } pci_read_config_dword(dev, notify + offsetof(struct virtio_pci_notify_cap_s, notify_off_multiplier), &vpdev->notify_off_multiplier); vpdev->notify = virtio_pci_map_capability(vpdev, notify, &vpdev->notify_len); if (vpdev->notify == NULL) { return -EINVAL; } if (device != 0) { vpdev->device = virtio_pci_map_capability(vpdev, device, &vpdev->device_len); if (vpdev->device == NULL) { return -EINVAL; } } return OK; } /**************************************************************************** * Name: virtio_pci_modern_probe ****************************************************************************/ int virtio_pci_modern_probe(FAR struct pci_device_s *dev) { FAR struct virtio_pci_device_s *vpdev = dev->priv; FAR struct virtio_device *vdev = &vpdev->vdev; int ret; /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ if (dev->device < 0x1000 || dev->device > 0x107f) { return -ENODEV; } vpdev->ops = &g_virtio_pci_modern_ops; vdev->func = &g_virtio_pci_dispatch; vdev->role = VIRTIO_DEV_DRIVER; ret = virtio_pci_init_device(vpdev); if (ret < 0) { if (ret != -ENODEV) { pcierr("Virtio pci modern device init failed, ret=%d\n", ret); } } return ret; }