From: Will Deacon willdeacon@google.com

If we detect the VIRTIO_F_IOMMU_PLATFORM transport feature for a device, then expose it in the device features.

Signed-off-by: Will Deacon willdeacon@google.com [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) paul.liu@linaro.org Cc: Bin Meng bmeng.cn@gmail.com Link: https://android.googlesource.com/platform/external/u-boot/+/9693bd26bfcfe77d... --- drivers/virtio/virtio-uclass.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/virtio/virtio-uclass.c b/drivers/virtio/virtio-uclass.c index de9bc90359..b3fb3dbfad 100644 --- a/drivers/virtio/virtio-uclass.c +++ b/drivers/virtio/virtio-uclass.c @@ -336,7 +336,7 @@ static int virtio_uclass_child_pre_probe(struct udevice *vdev) /* Transport features always preserved to pass to finalize_features */ for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) if ((device_features & (1ULL << i)) && - (i == VIRTIO_F_VERSION_1)) + (i == VIRTIO_F_VERSION_1 || i == VIRTIO_F_IOMMU_PLATFORM)) __virtio_set_bit(vdev->parent, i);

debug("(%s) final negotiated features supported %016llx\n",

From: Will Deacon willdeacon@google.com

The pages backing the virtqueues for virtio PCI devices are not freed on reset, despite the virtqueue structure being freed as part of the driver '->priv_auto' destruction at ->remove() time.

Call virtio_pci_del_vqs() from virtio_pci_reset() to free the virtqueue pages before freeing the virtqueue structure itself.

Signed-off-by: Will Deacon willdeacon@google.com [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) paul.liu@linaro.org Cc: Bin Meng bmeng.cn@gmail.com Link: https://android.googlesource.com/platform/external/u-boot/+/5ed54ccd83cbffd0... --- drivers/virtio/virtio_pci_modern.c | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index cfde4007f5..3cdc2d2d6f 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -218,25 +218,6 @@ static int virtio_pci_set_status(struct udevice *udev, u8 status) return 0; }

-static int virtio_pci_reset(struct udevice *udev) -{ - struct virtio_pci_priv *priv = dev_get_priv(udev); - - /* 0 status means a reset */ - iowrite8(0, &priv->common->device_status); - - /* - * After writing 0 to device_status, the driver MUST wait for a read - * of device_status to return 0 before reinitializing the device. - * This will flush out the status write, and flush in device writes, - * including MSI-X interrupts, if any. - */ - while (ioread8(&priv->common->device_status)) - udelay(1000); - - return 0; -} - static int virtio_pci_get_features(struct udevice *udev, u64 *features) { struct virtio_pci_priv *priv = dev_get_priv(udev); @@ -363,6 +344,25 @@ static int virtio_pci_find_vqs(struct udevice *udev, unsigned int nvqs, return 0; }

+static int virtio_pci_reset(struct udevice *udev) +{ + struct virtio_pci_priv *priv = dev_get_priv(udev); + + /* 0 status means a reset */ + iowrite8(0, &priv->common->device_status); + + /* + * After writing 0 to device_status, the driver MUST wait for a read + * of device_status to return 0 before reinitializing the device. + * This will flush out the status write, and flush in device writes, + * including MSI-X interrupts, if any. + */ + while (ioread8(&priv->common->device_status)) + udelay(1000); + + return virtio_pci_del_vqs(udev); +} + static int virtio_pci_notify(struct udevice *udev, struct virtqueue *vq) { struct virtio_pci_priv *priv = dev_get_priv(udev);

From: Will Deacon willdeacon@google.com

In preparation for explicit bouncing of virtqueue pages for devices advertising the VIRTIO_F_IOMMU_PLATFORM feature, introduce a couple of wrappers around virtqueue allocation and freeing operations, ensuring that buffers are handled in terms of page-size units.

Signed-off-by: Will Deacon willdeacon@google.com [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) paul.liu@linaro.org Cc: Bin Meng bmeng.cn@gmail.com Link: https://android.googlesource.com/platform/external/u-boot/+/b4bb5227d4cf4fdf... --- drivers/virtio/virtio_ring.c | 24 ++++++++++++++++++++---- include/virtio_ring.h | 16 +++++++++------- 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index f71bab7847..5aeb13fd59 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -15,6 +15,17 @@ #include <virtio_ring.h> #include <linux/bug.h> #include <linux/compat.h> +#include <linux/kernel.h> + +static void *virtio_alloc_pages(struct udevice *vdev, u32 npages) +{ + return memalign(PAGE_SIZE, npages * PAGE_SIZE); +} + +static void virtio_free_pages(struct udevice *vdev, void *ptr, u32 npages) +{ + free(ptr); +}

static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i, struct virtio_sg *sg, u16 flags) @@ -271,6 +282,8 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, unsigned int vring_align, struct udevice *udev) { + struct virtio_dev_priv *uc_priv = dev_get_uclass_priv(udev); + struct udevice *vdev = uc_priv->vdev; struct virtqueue *vq; void *queue = NULL; struct vring vring; @@ -283,7 +296,9 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num,

/* TODO: allocate each queue chunk individually */ for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) { - queue = memalign(PAGE_SIZE, vring_size(num, vring_align)); + size_t sz = vring_size(num, vring_align); + + queue = virtio_alloc_pages(vdev, DIV_ROUND_UP(sz, PAGE_SIZE)); if (queue) break; } @@ -293,7 +308,7 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num,

if (!queue) { /* Try to get a single page. You are my only hope! */ - queue = memalign(PAGE_SIZE, vring_size(num, vring_align)); + queue = virtio_alloc_pages(vdev, 1); } if (!queue) return NULL; @@ -303,7 +318,7 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num,

vq = __vring_new_virtqueue(index, vring, udev); if (!vq) { - free(queue); + virtio_free_pages(vdev, queue, DIV_ROUND_UP(vring.size, PAGE_SIZE)); return NULL; } debug("(%s): created vring @ %p for vq @ %p with num %u\n", udev->name, @@ -314,7 +329,8 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num,

void vring_del_virtqueue(struct virtqueue *vq) { - free(vq->vring.desc); + virtio_free_pages(vq->vdev, vq->vring.desc, + DIV_ROUND_UP(vq->vring.size, PAGE_SIZE)); free(vq->vring_desc_shadow); list_del(&vq->list); free(vq); diff --git a/include/virtio_ring.h b/include/virtio_ring.h index c77c212cff..8f8a55c7bd 100644 --- a/include/virtio_ring.h +++ b/include/virtio_ring.h @@ -86,6 +86,7 @@ struct vring_used {

struct vring { unsigned int num; + size_t size; struct vring_desc *desc; struct vring_avail *avail; struct vring_used *used; @@ -137,23 +138,24 @@ struct virtqueue { #define vring_used_event(vr) ((vr)->avail->ring[(vr)->num]) #define vring_avail_event(vr) (*(__virtio16 *)&(vr)->used->ring[(vr)->num])

+static inline unsigned int vring_size(unsigned int num, unsigned long align) +{ + return ((sizeof(struct vring_desc) * num + + sizeof(__virtio16) * (3 + num) + align - 1) & ~(align - 1)) + + sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num; +} + static inline void vring_init(struct vring *vr, unsigned int num, void *p, unsigned long align) { vr->num = num; + vr->size = vring_size(num, align); vr->desc = p; vr->avail = p + num * sizeof(struct vring_desc); vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(__virtio16) + align - 1) & ~(align - 1)); }

-static inline unsigned int vring_size(unsigned int num, unsigned long align) -{ - return ((sizeof(struct vring_desc) * num + - sizeof(__virtio16) * (3 + num) + align - 1) & ~(align - 1)) + - sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num; -} - /* * The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX. * Assuming a given event_idx value from the other side, if we have just

From: Will Deacon willdeacon@google.com

Move the attach and detach logic for manipulating vring descriptors out into their own functions so that we can later extend these to bounce the data for devices with VIRTIO_F_IOMMU_PLATFORM set.

Signed-off-by: Will Deacon willdeacon@google.com [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) paul.liu@linaro.org Cc: Bin Meng bmeng.cn@gmail.com Link: https://android.googlesource.com/platform/external/u-boot/+/f73258a4bfe968c5... --- drivers/virtio/virtio_ring.c | 6 ++++++ 1 file changed, 6 insertions(+)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 5aeb13fd59..1bd19add75 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -47,6 +47,10 @@ static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i, return desc_shadow->next; }

+static void virtqueue_detach_desc(struct virtqueue *vq, unsigned int idx) +{ +} + int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[], unsigned int out_sgs, unsigned int in_sgs) { @@ -165,10 +169,12 @@ static void detach_buf(struct virtqueue *vq, unsigned int head) i = head;

while (vq->vring_desc_shadow[i].flags & VRING_DESC_F_NEXT) { + virtqueue_detach_desc(vq, i); i = vq->vring_desc_shadow[i].next; vq->num_free++; }

+ virtqueue_detach_desc(vq, i); vq->vring_desc_shadow[i].next = vq->free_head; vq->free_head = head;

From: Will Deacon willdeacon@google.com

In preparation for bouncing virtio data for devices advertising the VIRTIO_F_IOMMU_PLATFORM feature, allocate an array of bounce buffer structures in the vring, one per descriptor.

Signed-off-by: Will Deacon willdeacon@google.com [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) paul.liu@linaro.org Cc: Bin Meng bmeng.cn@gmail.com Link: https://android.googlesource.com/platform/external/u-boot/+/3e052749e7c50c4c... --- drivers/virtio/virtio_ring.c | 25 ++++++++++++++++++++----- include/virtio_ring.h | 5 ++++- 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1bd19add75..de75786ca7 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -6,6 +6,7 @@ * virtio ring implementation */

+#include <bouncebuf.h> #include <common.h> #include <dm.h> #include <log.h> @@ -292,6 +293,7 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, struct udevice *vdev = uc_priv->vdev; struct virtqueue *vq; void *queue = NULL; + struct bounce_buffer *bbs = NULL; struct vring vring;

/* We assume num is a power of 2 */ @@ -320,17 +322,29 @@ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, return NULL;

memset(queue, 0, vring_size(num, vring_align)); - vring_init(&vring, num, queue, vring_align);

- vq = __vring_new_virtqueue(index, vring, udev); - if (!vq) { - virtio_free_pages(vdev, queue, DIV_ROUND_UP(vring.size, PAGE_SIZE)); - return NULL; + if (virtio_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) { + bbs = calloc(num, sizeof(*bbs)); + if (!bbs) + goto err_free_queue; } + + vring_init(&vring, num, queue, vring_align, bbs); + + vq = __vring_new_virtqueue(index, vring, udev); + if (!vq) + goto err_free_bbs; + debug("(%s): created vring @ %p for vq @ %p with num %u\n", udev->name, queue, vq, num);

return vq; + +err_free_bbs: + free(bbs); +err_free_queue: + virtio_free_pages(vdev, queue, DIV_ROUND_UP(vring.size, PAGE_SIZE)); + return NULL; }

void vring_del_virtqueue(struct virtqueue *vq) @@ -339,6 +353,7 @@ void vring_del_virtqueue(struct virtqueue *vq) DIV_ROUND_UP(vq->vring.size, PAGE_SIZE)); free(vq->vring_desc_shadow); list_del(&vq->list); + free(vq->vring.bouncebufs); free(vq); }

diff --git a/include/virtio_ring.h b/include/virtio_ring.h index 8f8a55c7bd..e8e91044a2 100644 --- a/include/virtio_ring.h +++ b/include/virtio_ring.h @@ -87,6 +87,7 @@ struct vring_used { struct vring { unsigned int num; size_t size; + struct bounce_buffer *bouncebufs; struct vring_desc *desc; struct vring_avail *avail; struct vring_used *used; @@ -146,10 +147,12 @@ static inline unsigned int vring_size(unsigned int num, unsigned long align) }

static inline void vring_init(struct vring *vr, unsigned int num, void *p, - unsigned long align) + unsigned long align, + struct bounce_buffer *bouncebufs) { vr->num = num; vr->size = vring_size(num, align); + vr->bouncebufs = bouncebufs; vr->desc = p; vr->avail = p + num * sizeof(struct vring_desc); vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] +

From: Will Deacon willdeacon@google.com

Devices advertising the VIRTIO_F_IOMMU_PLATFORM feature require platform-specific handling to configure their DMA transactions.

When handling virtio descriptors for such a device, use bounce buffers to ensure that the underlying buffers are always aligned to and padded to PAGE_SIZE in preparation for platform specific handling at page granularity.

Signed-off-by: Will Deacon willdeacon@google.com [ Paul: pick from the Android tree. Rebase to the upstream ] Signed-off-by: Ying-Chun Liu (PaulLiu) paul.liu@linaro.org Cc: Bin Meng bmeng.cn@gmail.com Link: https://android.googlesource.com/platform/external/u-boot/+/1eff171e613ee67d... --- drivers/virtio/virtio_ring.c | 48 +++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index de75786ca7..c9adcce5c0 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -28,14 +28,51 @@ static void virtio_free_pages(struct udevice *vdev, void *ptr, u32 npages) free(ptr); }

+static int __bb_force_page_align(struct bounce_buffer *state) +{ + const ulong align_mask = PAGE_SIZE - 1; + + if ((ulong)state->user_buffer & align_mask) + return 0; + + if (state->len != state->len_aligned) + return 0; + + return 1; +} + static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i, struct virtio_sg *sg, u16 flags) { struct vring_desc_shadow *desc_shadow = &vq->vring_desc_shadow[i]; struct vring_desc *desc = &vq->vring.desc[i]; + void *addr; + + if (IS_ENABLED(CONFIG_BOUNCE_BUFFER) && vq->vring.bouncebufs) { + struct bounce_buffer *bb = &vq->vring.bouncebufs[i]; + unsigned int bbflags; + int ret; + + if (flags & VRING_DESC_F_WRITE) + bbflags = GEN_BB_WRITE; + else + bbflags = GEN_BB_READ; + + ret = bounce_buffer_start_extalign(bb, sg->addr, sg->length, + bbflags, PAGE_SIZE, + __bb_force_page_align); + if (ret) { + debug("%s: failed to allocate bounce buffer (length 0x%zx)\n", + vq->vdev->name, sg->length); + } + + addr = bb->bounce_buffer; + } else { + addr = sg->addr; + }

/* Update the shadow descriptor. */ - desc_shadow->addr = (u64)(uintptr_t)sg->addr; + desc_shadow->addr = (u64)(uintptr_t)addr; desc_shadow->len = sg->length; desc_shadow->flags = flags;

@@ -50,6 +87,15 @@ static unsigned int virtqueue_attach_desc(struct virtqueue *vq, unsigned int i,

static void virtqueue_detach_desc(struct virtqueue *vq, unsigned int idx) { + struct vring_desc *desc = &vq->vring.desc[idx]; + struct bounce_buffer *bb; + + if (!IS_ENABLED(CONFIG_BOUNCE_BUFFER) || !vq->vring.bouncebufs) + return; + + bb = &vq->vring.bouncebufs[idx]; + bounce_buffer_stop(bb); + desc->addr = cpu_to_virtio64(vq->vdev, (u64)(uintptr_t)bb->user_buffer); }

int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[],