On Thu, Aug 21, 2025 at 02:46:38PM +0800, Jason Wang wrote:
> This patch introduces map operations for virtio device. Virtio use to
> use DMA API which is not necessarily the case since some devices
used to use
> doesn't do DMA. Instead of using tricks and abusing DMA API, let's
> simply abstract the current mapping logic into a virtio specific
> mapping operations. For the device or transport that doesn't do DMA,
> they can implement their own mapping logic without the need to trick
> DMA core. In this case the mapping metadata is opaque to the virtio
> core that will be passed back to the transport or device specific map
> operations. For other devices, DMA API will still be used, so map
> token will still be the dma device to minimize the changeset and
> performance impact.
>
> The mapping operations are abstracted as a independent structure
> instead of reusing virtio_config_ops. This allows the transport can
> simply reuse the structure for lower layers like vDPA.
>
> A set of new mapping helpers were introduced for the device that want
> to do mapping by themselves.
>
> Signed-off-by: Jason Wang <jasowang@xxxxxxxxxx>
> ---
> drivers/virtio/virtio_ring.c | 211 +++++++++++++++++++++++++++-------
> drivers/virtio/virtio_vdpa.c | 3 +
> include/linux/virtio.h | 25 ++++
> include/linux/virtio_config.h | 72 ++++++++++++
> 4 files changed, 269 insertions(+), 42 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 46515b017ccb..f91a432b3e53 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -297,8 +297,14 @@ size_t virtio_max_dma_size(const struct virtio_device *vdev)
> {
> size_t max_segment_size = SIZE_MAX;
>
> - if (vring_use_map_api(vdev))
> - max_segment_size = dma_max_mapping_size(vdev->dev.parent);
> + if (vring_use_map_api(vdev)) {
> + if (vdev->map) {
> + max_segment_size =
> + vdev->map->max_mapping_size(vdev->vmap);
> + } else
> + max_segment_size =
> + dma_max_mapping_size(vdev->dev.parent);
> + }
>
> return max_segment_size;
> }
> @@ -309,8 +315,8 @@ static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
> union virtio_map map)
> {
> if (vring_use_map_api(vdev)) {
> - return dma_alloc_coherent(map.dma_dev, size,
> - map_handle, flag);
> + return virtqueue_map_alloc_coherent(vdev, map, size,
> + map_handle, flag);
> } else {
> void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
>
> @@ -343,7 +349,8 @@ static void vring_free_queue(struct virtio_device *vdev, size_t size,
> union virtio_map map)
> {
> if (vring_use_map_api(vdev))
> - dma_free_coherent(map.dma_dev, size, queue, map_handle);
> + virtqueue_map_free_coherent(vdev, map, size,
> + queue, map_handle);
> else
> free_pages_exact(queue, PAGE_ALIGN(size));
> }
> @@ -358,6 +365,20 @@ static struct device *vring_dma_dev(const struct vring_virtqueue *vq)
> return vq->map.dma_dev;
> }
>
> +static int vring_mapping_error(const struct vring_virtqueue *vq,
> + dma_addr_t addr)
> +{
> + struct virtio_device *vdev = vq->vq.vdev;
> +
> + if (!vq->use_map_api)
> + return 0;
> +
> + if (vdev->map)
> + return vdev->map->mapping_error(vq->map, addr);
> + else
> + return dma_mapping_error(vring_dma_dev(vq), addr);
> +}
> +
> /* Map one sg entry. */
> static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg,
> enum dma_data_direction direction, dma_addr_t *addr,
> @@ -387,11 +408,11 @@ static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist
> * the way it expects (we don't guarantee that the scatterlist
> * will exist for the lifetime of the mapping).
> */
> - *addr = dma_map_page(vring_dma_dev(vq),
> - sg_page(sg), sg->offset, sg->length,
> - direction);
> + *addr = virtqueue_map_page_attrs(&vq->vq, sg_page(sg),
> + sg->offset, sg->length,
> + direction, 0);
>
> - if (dma_mapping_error(vring_dma_dev(vq), *addr))
> + if (vring_mapping_error(vq, *addr))
> return -ENOMEM;
>
> return 0;
> @@ -408,15 +429,6 @@ static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
> size, direction, 0);
> }
>
> -static int vring_mapping_error(const struct vring_virtqueue *vq,
> - dma_addr_t addr)
> -{
> - if (!vq->use_map_api)
> - return 0;
> -
> - return dma_mapping_error(vring_dma_dev(vq), addr);
> -}
> -
> static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
> {
> vq->vq.num_free = num;
> @@ -453,11 +465,12 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
> } else if (!vring_need_unmap_buffer(vq, extra))
> goto out;
>
> - dma_unmap_page(vring_dma_dev(vq),
> - extra->addr,
> - extra->len,
> - (flags & VRING_DESC_F_WRITE) ?
> - DMA_FROM_DEVICE : DMA_TO_DEVICE);
> + virtqueue_unmap_page_attrs(&vq->vq,
> + extra->addr,
> + extra->len,
> + (flags & VRING_DESC_F_WRITE) ?
> + DMA_FROM_DEVICE : DMA_TO_DEVICE,
> + 0);
>
> out:
> return extra->next;
> @@ -1271,10 +1284,11 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
> } else if (!vring_need_unmap_buffer(vq, extra))
> return;
>
> - dma_unmap_page(vring_dma_dev(vq),
> - extra->addr, extra->len,
> - (flags & VRING_DESC_F_WRITE) ?
> - DMA_FROM_DEVICE : DMA_TO_DEVICE);
> + virtqueue_unmap_page_attrs(&vq->vq,
> + extra->addr, extra->len,
> + (flags & VRING_DESC_F_WRITE) ?
> + DMA_FROM_DEVICE : DMA_TO_DEVICE,
> + 0);
> }
>
> static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
> @@ -2433,7 +2447,7 @@ struct device *virtqueue_dma_dev(struct virtqueue *_vq)
> {
> struct vring_virtqueue *vq = to_vvq(_vq);
>
> - if (vq->use_map_api)
> + if (vq->use_map_api && !_vq->vdev->map)
> return vq->map.dma_dev;
> else
> return NULL;
> @@ -3123,6 +3137,107 @@ const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
> }
> EXPORT_SYMBOL_GPL(virtqueue_get_vring);
>
> +/**
> + * virtqueue_map_alloc_coherent - alloc coherent mapping
> + * @vdev: the virtio device we are talking to
> + * @map: metadata for performing mapping
> + * @size: the size of the buffer
> + * @map_handle: the pointer to the mapped address
> + * @gfp: allocation flag (GFP_XXX)
> + *
> + * return virtual address or NULL on error
> + */
> +void *virtqueue_map_alloc_coherent(struct virtio_device *vdev,
> + union virtio_map map,
> + size_t size, dma_addr_t *map_handle,
> + gfp_t gfp)
> +{
> + if (vdev->map)
> + return vdev->map->alloc(map, size,
> + map_handle, gfp);
> + else
> + return dma_alloc_coherent(map.dma_dev, size,
> + map_handle, gfp);
> +}
> +EXPORT_SYMBOL_GPL(virtqueue_map_alloc_coherent);
> +
> +/**
> + * virtqueue_map_free_coherent - free coherent mapping
> + * @vdev: the virtio device we are talking to
> + * @map: metadata for performing mapping
> + * @size: the size of the buffer
> + * @map_handle: the mapped address that needs to be freed
> + *
> + */
> +void virtqueue_map_free_coherent(struct virtio_device *vdev,
> + union virtio_map map, size_t size, void *vaddr,
> + dma_addr_t map_handle)
> +{
> + if (vdev->map)
> + vdev->map->free(map, size, vaddr,
> + map_handle, 0);
> + else
> + dma_free_coherent(map.dma_dev, size, vaddr, map_handle);
> +}
> +EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent);
> +
> +/**
> + * virtqueue_map_page_attrs - map a page to the device
> + * @_vq: the virtqueue we are talking to
> + * @page: the page that will be mapped by the device
> + * @offset: the offset in the page for a buffer
> + * @size: the buffer size
> + * @dir: mapping direction
> + * @attrs: mapping attributes
> + *
> + * Returns mapped address. Caller should check that by virtqueue_mapping_error().
> + */
> +dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
> + struct page *page,
> + unsigned long offset,
> + size_t size,
> + enum dma_data_direction dir,
> + unsigned long attrs)
> +{
> + const struct vring_virtqueue *vq = to_vvq(_vq);
> + struct virtio_device *vdev = _vq->vdev;
> +
> + if (vdev->map)
> + return vdev->map->map_page(vq->map,
> + page, offset, size,
> + dir, attrs);
> +
> + return dma_map_page_attrs(vring_dma_dev(vq),
> + page, offset, size,
> + dir, attrs);
> +}
> +EXPORT_SYMBOL_GPL(virtqueue_map_page_attrs);
> +
> +/**
> + * virtqueue_unmap_page_attrs - map a page to the device
> + * @_vq: the virtqueue we are talking to
> + * @map_handle: the mapped address
> + * @size: the buffer size
> + * @dir: mapping direction
> + * @attrs: unmapping attributes
> + */
> +void virtqueue_unmap_page_attrs(const struct virtqueue *_vq,
> + dma_addr_t map_handle,
> + size_t size, enum dma_data_direction dir,
> + unsigned long attrs)
> +{
> + const struct vring_virtqueue *vq = to_vvq(_vq);
> + struct virtio_device *vdev = _vq->vdev;
> +
> + if (vdev->map)
> + vdev->map->unmap_page(vq->map,
> + map_handle, size, dir, attrs);
> + else
> + dma_unmap_page_attrs(vring_dma_dev(vq), map_handle,
> + size, dir, attrs);
> +}
> +EXPORT_SYMBOL_GPL(virtqueue_unmap_page_attrs);
> +
> /**
> * virtqueue_map_single_attrs - map DMA for _vq
> * @_vq: the struct virtqueue we're talking about.
> @@ -3134,7 +3249,7 @@ EXPORT_SYMBOL_GPL(virtqueue_get_vring);
> * The caller calls this to do dma mapping in advance. The DMA address can be
> * passed to this _vq when it is in pre-mapped mode.
> *
> - * return DMA address. Caller should check that by virtqueue_mapping_error().
> + * return mapped address. Caller should check that by virtqueue_mapping_error().
> */
> dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr,
> size_t size,
> @@ -3153,8 +3268,8 @@ dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr,
> "rejecting DMA map of vmalloc memory\n"))
> return DMA_MAPPING_ERROR;
>
> - return dma_map_page_attrs(vring_dma_dev(vq), virt_to_page(ptr),
> - offset_in_page(ptr), size, dir, attrs);
> + return virtqueue_map_page_attrs(&vq->vq, virt_to_page(ptr),
> + offset_in_page(ptr), size, dir, attrs);
> }
> EXPORT_SYMBOL_GPL(virtqueue_map_single_attrs);
>
> @@ -3179,12 +3294,12 @@ void virtqueue_unmap_single_attrs(const struct virtqueue *_vq,
> if (!vq->use_map_api)
> return;
>
> - dma_unmap_page_attrs(vring_dma_dev(vq), addr, size, dir, attrs);
> + virtqueue_unmap_page_attrs(_vq, addr, size, dir, attrs);
> }
> EXPORT_SYMBOL_GPL(virtqueue_unmap_single_attrs);
>
> /**
> - * virtqueue_map_mapping_error - check dma address
> + * virtqueue_mapping_error - check dma address
> * @_vq: the struct virtqueue we're talking about.
> * @addr: DMA address
> *
> @@ -3194,10 +3309,7 @@ int virtqueue_map_mapping_error(const struct virtqueue *_vq, dma_addr_t addr)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
>
> - if (!vq->use_map_api)
> - return 0;
> -
> - return dma_mapping_error(vring_dma_dev(vq), addr);
> + return vring_mapping_error(vq, addr);
> }
> EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error);
>
> @@ -3214,11 +3326,15 @@ EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error);
> bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> + struct virtio_device *vdev = _vq->vdev;
>
> if (!vq->use_map_api)
> return false;
>
> - return dma_need_sync(vring_dma_dev(vq), addr);
> + if (vdev->map)
> + return vdev->map->need_sync(vq->map, addr);
> + else
> + return dma_need_sync(vring_dma_dev(vq), addr);
> }
> EXPORT_SYMBOL_GPL(virtqueue_map_need_sync);
>
> @@ -3240,12 +3356,17 @@ void virtqueue_map_sync_single_range_for_cpu(const struct virtqueue *_vq,
> enum dma_data_direction dir)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> - struct device *dev = vring_dma_dev(vq);
> + struct virtio_device *vdev = _vq->vdev;
>
> if (!vq->use_map_api)
> return;
>
> - dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
> + if (vdev->map)
> + vdev->map->sync_single_for_cpu(vq->map,
> + addr + offset, size, dir);
> + else
> + dma_sync_single_range_for_cpu(vring_dma_dev(vq),
> + addr, offset, size, dir);
> }
> EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_cpu);
>
> @@ -3266,12 +3387,18 @@ void virtqueue_map_sync_single_range_for_device(const struct virtqueue *_vq,
> enum dma_data_direction dir)
> {
> const struct vring_virtqueue *vq = to_vvq(_vq);
> - struct device *dev = vring_dma_dev(vq);
> + struct virtio_device *vdev = _vq->vdev;
>
> if (!vq->use_map_api)
> return;
>
> - dma_sync_single_range_for_device(dev, addr, offset, size, dir);
> + if (vdev->map)
> + vdev->map->sync_single_for_device(vq->map,
> + addr + offset,
> + size, dir);
> + else
> + dma_sync_single_range_for_device(vring_dma_dev(vq), addr,
> + offset, size, dir);
> }
> EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_device);
>
> diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
> index dc557aa7c825..d4be689e3626 100644
> --- a/drivers/virtio/virtio_vdpa.c
> +++ b/drivers/virtio/virtio_vdpa.c
> @@ -195,6 +195,9 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
> goto error_new_virtqueue;
> }
>
> + if (index == 0)
> + vdev->vmap = map;
> +
> vq->num_max = max_num;
>
> /* Setup virtqueue callback */
> diff --git a/include/linux/virtio.h b/include/linux/virtio.h
> index b4ba1a99e5ab..3386a4a8d06b 100644
> --- a/include/linux/virtio.h
> +++ b/include/linux/virtio.h
> @@ -166,9 +166,11 @@ struct virtio_device {
> struct virtio_device_id id;
> const struct virtio_config_ops *config;
> const struct vringh_config_ops *vringh_config;
> + const struct virtio_map_ops *map;
> struct list_head vqs;
> VIRTIO_DECLARE_FEATURES(features);
> void *priv;
> + union virtio_map vmap;
> #ifdef CONFIG_VIRTIO_DEBUG
> struct dentry *debugfs_dir;
> u64 debugfs_filter_features[VIRTIO_FEATURES_DWORDS];
> @@ -267,6 +269,29 @@ void unregister_virtio_driver(struct virtio_driver *drv);
> module_driver(__virtio_driver, register_virtio_driver, \
> unregister_virtio_driver)
>
> +
> +void *virtqueue_map_alloc_coherent(struct virtio_device *vdev,
> + union virtio_map mapping_token,
> + size_t size, dma_addr_t *dma_handle,
> + gfp_t gfp);
> +
> +void virtqueue_map_free_coherent(struct virtio_device *vdev,
> + union virtio_map mapping_token,
> + size_t size, void *vaddr,
> + dma_addr_t dma_handle);
> +
> +dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq,
> + struct page *page,
> + unsigned long offset,
> + size_t size,
> + enum dma_data_direction dir,
> + unsigned long attrs);
> +
> +void virtqueue_unmap_page_attrs(const struct virtqueue *_vq,
> + dma_addr_t dma_handle,
> + size_t size, enum dma_data_direction dir,
> + unsigned long attrs);
> +
> dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr, size_t size,
> enum dma_data_direction dir, unsigned long attrs);
> void virtqueue_unmap_single_attrs(const struct virtqueue *_vq, dma_addr_t addr,
> diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h
> index 8bf156dde554..52319dc5d5e0 100644
> --- a/include/linux/virtio_config.h
> +++ b/include/linux/virtio_config.h
> @@ -139,6 +139,78 @@ struct virtio_config_ops {
> int (*enable_vq_after_reset)(struct virtqueue *vq);
> };
>
> +/**
> + * struct virtio_map_ops - operations for mapping buffer for a virtio device
> + * Note: For transport that has its own mapping logic it must
> + * implements all of the operations
> + * @map_page: map a buffer to the device
> + * map: metadata for performing mapping
> + * page: the page that will be mapped by the device
> + * offset: the offset in the page for a buffer
> + * size: the buffer size
> + * dir: mapping direction
> + * attrs: mapping attributes
> + * Returns: the mapped address
> + * @unmap_page: unmap a buffer from the device
> + * map: device specific mapping map
> + * map_handle: the mapped address
> + * size: the buffer size
> + * dir: mapping direction
> + * attrs: unmapping attributes
> + * @sync_single_for_cpu: sync a single buffer from device to cpu
> + * map: metadata for performing mapping
> + * map_handle: the mapping address to sync
> + * size: the size of the buffer
> + * dir: synchronization direction
> + * @sync_single_for_device: sync a single buffer from cpu to device
> + * map: metadata for performing mapping
> + * map_handle: the mapping address to sync
> + * size: the size of the buffer
> + * dir: synchronization direction
> + * @alloc: alloc a coherent buffer mapping
> + * map: metadata for performing mapping
> + * size: the size of the buffer
> + * map_handle: the mapping address to sync
> + * gfp: allocation flag (GFP_XXX)
> + * Returns: virtual address of the allocated buffer
> + * @free: free a coherent buffer mapping
> + * map: metadata for performing mapping
> + * size: the size of the buffer
> + * vaddr: virtual address of the buffer
> + * map_handle: the mapping address to sync
> + * attrs: unmapping attributes
> + * @need_sync: if the buffer needs synchronization
> + * map: metadata for performing mapping
> + * map_handle: the mapped address
> + * Returns: whether the buffer needs synchronization
> + * @mapping_error: if the mapping address is error
> + * map: metadata for performing mapping
> + * map_handle: the mapped address
> + * @max_mapping_size: get the maximum buffer size that can be mapped
> + * map: metadata for performing mapping
> + * Returns: the maximum buffer size that can be mapped
> + */
> +struct virtio_map_ops {
> + dma_addr_t (*map_page)(union virtio_map map, struct page *page,
> + unsigned long offset, size_t size,
> + enum dma_data_direction dir, unsigned long attrs);
> + void (*unmap_page)(union virtio_map map, dma_addr_t map_handle,
> + size_t size, enum dma_data_direction dir,
> + unsigned long attrs);
> + void (*sync_single_for_cpu)(union virtio_map map, dma_addr_t map_handle,
> + size_t size, enum dma_data_direction dir);
> + void (*sync_single_for_device)(union virtio_map map,
> + dma_addr_t map_handle, size_t size,
> + enum dma_data_direction dir);
> + void *(*alloc)(union virtio_map map, size_t size,
> + dma_addr_t *map_handle, gfp_t gfp);
> + void (*free)(union virtio_map map, size_t size, void *vaddr,
> + dma_addr_t map_handle, unsigned long attrs);
> + bool (*need_sync)(union virtio_map map, dma_addr_t map_handle);
> + int (*mapping_error)(union virtio_map map, dma_addr_t map_handle);
> + size_t (*max_mapping_size)(union virtio_map map);
> +};
> +
> /* If driver didn't advertise the feature, it will never appear. */
> void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
> unsigned int fbit);
> --
> 2.31.1