Re: [PATCH 1/3] arm: apple: nvme: Add SART support and RTKit buffer management

From: Janne Grunau j@jannau.net Date: Sun, 12 Jun 2022 18:00:27 +0200

The NVMe firmware in the macOS 13 beta blocks or crashes with u-boot's current minimal RTKit implementation. It does not provide buffers for the firmware's buffer requests. The ANS2 firmware included in macOS 11 and 12 tolerates this. The firmware included in the first macOS 13 beta requires buffers for the crashlog and ioreport endpoints to function.

In the case of the NVMe the buffers are physical memory. Access to physical memory is guarded by what Apple calls SART. Import m1n1's SART driver (exclusively used for the NVMe controller). Implement buffer management helpers for RTKit. These are generic since other devices (none in u-boot so far) require different handling.

Signed-off-by: Janne Grunau j@jannau.net

Thanks for taking care of this.

Reviewed-by: Mark Kettenis kettenis@openbsd.org Tested-by: Mark Kettenis kettenis@openbsd.org

---

arch/arm/include/asm/arch-apple/rtkit.h | 22 ++- arch/arm/include/asm/arch-apple/sart.h | 22 +++ arch/arm/mach-apple/Makefile | 1 + arch/arm/mach-apple/rtkit.c | 159 +++++++++++++--- arch/arm/mach-apple/sart.c | 230 ++++++++++++++++++++++++ drivers/nvme/nvme_apple.c | 72 +++++++- 6 files changed, 474 insertions(+), 32 deletions(-) create mode 100644 arch/arm/include/asm/arch-apple/sart.h create mode 100644 arch/arm/mach-apple/sart.c

diff --git a/arch/arm/include/asm/arch-apple/rtkit.h b/arch/arm/include/asm/arch-apple/rtkit.h index 51f77f298c09..eff18ddb9d21 100644 --- a/arch/arm/include/asm/arch-apple/rtkit.h +++ b/arch/arm/include/asm/arch-apple/rtkit.h @@ -7,5 +7,23 @@ #define APPLE_RTKIT_PWR_STATE_QUIESCED 0x10 #define APPLE_RTKIT_PWR_STATE_ON 0x20

-int apple_rtkit_init(struct mbox_chan *); -int apple_rtkit_shutdown(struct mbox_chan *, int); +struct apple_rtkit_buffer {

• void *buffer;
• u64 dva;
• size_t size;
• bool is_mapped;

+};

+typedef int (*apple_rtkit_shmem_setup)(void *cookie,

• 		       struct apple_rtkit_buffer *buf);
  

+typedef void (*apple_rtkit_shmem_destroy)(void *cookie,

• 			  struct apple_rtkit_buffer *buf);
  

+struct apple_rtkit;

+struct apple_rtkit *apple_rtkit_init(struct mbox_chan *chan, void *cookie,

• 		     apple_rtkit_shmem_setup shmem_setup,
  

• 		     apple_rtkit_shmem_destroy shmem_destroy);
  

+void apple_rtkit_free(struct apple_rtkit *rtk); +int apple_rtkit_boot(struct apple_rtkit *rtk); +int apple_rtkit_shutdown(struct apple_rtkit *rtk, int pwrstate); diff --git a/arch/arm/include/asm/arch-apple/sart.h b/arch/arm/include/asm/arch-apple/sart.h new file mode 100644 index 000000000000..b99bdeafc0b3 --- /dev/null +++ b/arch/arm/include/asm/arch-apple/sart.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT

• • The sart code is copied from m1n1 (https://github.com/AsahiLinux/m1n1) and
• • licensed as MIT.
• • (C) Copyright 2022 The Asahi Linux Contributors
• */

+#ifndef SART_H +#define SART_H

+#include <dm/ofnode.h>

+struct apple_sart;

+struct apple_sart *sart_init(ofnode node); +void sart_free(struct apple_sart *sart);

+bool sart_add_allowed_region(struct apple_sart *sart, void *paddr, size_t sz); +bool sart_remove_allowed_region(struct apple_sart *sart, void *paddr, size_t sz);

+#endif diff --git a/arch/arm/mach-apple/Makefile b/arch/arm/mach-apple/Makefile index 52f30a777b2c..50b465b9473f 100644 --- a/arch/arm/mach-apple/Makefile +++ b/arch/arm/mach-apple/Makefile @@ -3,3 +3,4 @@ obj-y += board.o obj-y += lowlevel_init.o obj-y += rtkit.o +obj-$(CONFIG_NVME_APPLE) += sart.o diff --git a/arch/arm/mach-apple/rtkit.c b/arch/arm/mach-apple/rtkit.c index 2dcb8bdd3e44..da7771844230 100644 --- a/arch/arm/mach-apple/rtkit.c +++ b/arch/arm/mach-apple/rtkit.c @@ -17,6 +17,7 @@ #define APPLE_RTKIT_EP_SYSLOG 2 #define APPLE_RTKIT_EP_DEBUG 3 #define APPLE_RTKIT_EP_IOREPORT 4 +#define APPLE_RTKIT_EP_TRACEKIT 10

/* Messages for management endpoint. */ #define APPLE_RTKIT_MGMT_TYPE GENMASK(59, 52) @@ -51,7 +52,102 @@ #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK(51, 44) #define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK(41, 0)

-int apple_rtkit_init(struct mbox_chan *chan) +struct apple_rtkit {

• struct mbox_chan *chan;
• void *cookie;
• apple_rtkit_shmem_setup shmem_setup;
• apple_rtkit_shmem_destroy shmem_destroy;
• struct apple_rtkit_buffer syslog_buffer;
• struct apple_rtkit_buffer crashlog_buffer;
• struct apple_rtkit_buffer ioreport_buffer;

+};

+struct apple_rtkit *apple_rtkit_init(struct mbox_chan *chan, void *cookie,

• 		     apple_rtkit_shmem_setup shmem_setup,
  

• 		     apple_rtkit_shmem_destroy shmem_destroy)
  

+{

• struct apple_rtkit *rtk;
• rtk = calloc(sizeof(*rtk), 1);
• if (!rtk)

• return NULL;
  

• rtk->chan = chan;
• rtk->cookie = cookie;
• rtk->shmem_setup = shmem_setup;
• rtk->shmem_destroy = shmem_destroy;
• return rtk;

+}

+void apple_rtkit_free(struct apple_rtkit *rtk) +{

• if (rtk->shmem_destroy) {

• if (rtk->syslog_buffer.buffer)
  

• 	rtk->shmem_destroy(rtk->cookie, &rtk->syslog_buffer);
  

• if (rtk->crashlog_buffer.buffer)
  

• 	rtk->shmem_destroy(rtk->cookie, &rtk->crashlog_buffer);
  

• if (rtk->ioreport_buffer.buffer)
  

• 	rtk->shmem_destroy(rtk->cookie, &rtk->ioreport_buffer);
  

• }
• free(rtk);

+}

+static int rtkit_handle_buf_req(struct apple_rtkit *rtk, int endpoint, struct apple_mbox_msg *msg) +{

• struct apple_rtkit_buffer *buf;
• size_t num_4kpages;
• int ret;
• num_4kpages = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg->msg0);
• if (num_4kpages == 0) {

• printf("%s: unexpected request for buffer without size\n", __func__);
  

• return -1;
  

• }
• switch (endpoint) {
• case APPLE_RTKIT_EP_CRASHLOG:

• buf = &rtk->crashlog_buffer;
  

• break;
  

• case APPLE_RTKIT_EP_SYSLOG:

• buf = &rtk->syslog_buffer;
  

• break;
  

• case APPLE_RTKIT_EP_IOREPORT:

• buf = &rtk->ioreport_buffer;
  

• break;
  

• default:

• printf("%s: unexpected endpoint %d\n", __func__, endpoint);
  

• return -1;
  

• }
• buf->dva = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg->msg0);
• buf->size = num_4kpages << 12;
• buf->is_mapped = false;
• if (rtk->shmem_setup) {

• ret = rtk->shmem_setup(rtk->cookie, buf);
  

• if (ret < 0) {
  

• 	printf("%s: shmen_setup failed for endpoint %d\n", __func__,
  

• 	       endpoint);
  

• 	return ret;
  

• }
  

• }
• if (!buf->is_mapped) {

• msg->msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_BUFFER_REQUEST) |
  

• 		FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, num_4kpages) |
  

• 		FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, buf->dva);
  

• msg->msg1 = endpoint;
  

• return mbox_send(rtk->chan, msg);
  

• }
• return 0;

+}

+int apple_rtkit_boot(struct apple_rtkit *rtk) { struct apple_mbox_msg msg; int endpoints[256]; @@ -67,12 +163,12 @@ int apple_rtkit_init(struct mbox_chan *chan) msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE) | FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_ON); msg.msg1 = APPLE_RTKIT_EP_MGMT;

• ret = mbox_send(chan, &msg);

• ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret;

  /* Wait for protocol version negotiation message. */

• ret = mbox_recv(chan, &msg, 10000);

• ret = mbox_recv(rtk->chan, &msg, 10000); if (ret < 0) return ret;

@@ -108,13 +204,13 @@ int apple_rtkit_init(struct mbox_chan *chan) FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver) | FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver); msg.msg1 = APPLE_RTKIT_EP_MGMT;

• ret = mbox_send(chan, &msg);

• ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret;

wait_epmap: /* Wait for endpoint map message. */

• ret = mbox_recv(chan, &msg, 10000);

• ret = mbox_recv(rtk->chan, &msg, 10000); if (ret < 0) return ret;

@@ -145,7 +241,7 @@ wait_epmap: reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; msg.msg0 = reply; msg.msg1 = APPLE_RTKIT_EP_MGMT;

• ret = mbox_send(chan, &msg);

• ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret;

@@ -153,24 +249,33 @@ wait_epmap: goto wait_epmap;

for (i = 0; i < nendpoints; i++) {

• /* Don't start the syslog endpoint since we can't
  

•    easily handle its messages in U-Boot. */
  

• if (endpoints[i] == APPLE_RTKIT_EP_SYSLOG)
  

• /* Start only necessary endpoints. The syslog endpoint is
  

•  * particularly noisy and its message can't easily be handled
  

•  * within U-Boot.
  

•  */
  

• switch (endpoints[i]) {
  

• case APPLE_RTKIT_EP_MGMT:
  

• case APPLE_RTKIT_EP_SYSLOG:
  

• case APPLE_RTKIT_EP_DEBUG:
  

• case APPLE_RTKIT_EP_TRACEKIT:
  continue;
  

• default:
  

• 	break;
  

• }
  
  

  /* Request endpoint. */ msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_STARTEP) | FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoints[i]) | APPLE_RTKIT_MGMT_STARTEP_FLAG; msg.msg1 = APPLE_RTKIT_EP_MGMT;

• ret = mbox_send(chan, &msg);
  

• ret = mbox_send(rtk->chan, &msg);
  

  if (ret < 0) return ret; }

  pwrstate = APPLE_RTKIT_PWR_STATE_SLEEP; while (pwrstate != APPLE_RTKIT_PWR_STATE_ON) {

• ret = mbox_recv(chan, &msg, 1000000);
  

• ret = mbox_recv(rtk->chan, &msg, 100000);
  

  if (ret < 0) return ret;

@@ -180,20 +285,22 @@ wait_epmap: if (endpoint == APPLE_RTKIT_EP_CRASHLOG || endpoint == APPLE_RTKIT_EP_SYSLOG || endpoint == APPLE_RTKIT_EP_IOREPORT) {

• 	u64 addr = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg.msg0);
  

• 	u64 size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg.msg0);
  

• 	if (msgtype == APPLE_RTKIT_BUFFER_REQUEST && addr != 0)
  

• 	if (msgtype == APPLE_RTKIT_BUFFER_REQUEST) {
  

• 		ret = rtkit_handle_buf_req(rtk, endpoint, &msg);
  

• 		if (ret < 0)
  

• 			return ret;
  	continue;
  

• 	}
  

• }
  
  

• 	msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_BUFFER_REQUEST) |
  

• 		FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, size) |
  

• 		FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, addr);
  

• 	msg.msg1 = endpoint;
  

• 	ret = mbox_send(chan, &msg);
  

• 	if (ret < 0)
  

• 		return ret;
  

• 	continue;
  

• if (endpoint == APPLE_RTKIT_EP_IOREPORT) {
  

• 	// these two messages have to be ack-ed for proper startup
  

• 	if (msgtype == 0xc || msgtype == 0x8) {
  

• 		ret = mbox_send(rtk->chan, &msg);
  

• 		if (ret < 0)
  

• 			return ret;
  

• 		continue;
  

• 	}
  

  }

  if (endpoint != APPLE_RTKIT_EP_MGMT) {

@@ -211,7 +318,7 @@ wait_epmap: return 0; }

-int apple_rtkit_shutdown(struct mbox_chan *chan, int pwrstate) +int apple_rtkit_shutdown(struct apple_rtkit *rtk, int pwrstate) { struct apple_mbox_msg msg; int ret; @@ -219,11 +326,11 @@ int apple_rtkit_shutdown(struct mbox_chan *chan, int pwrstate) msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE) | FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, pwrstate); msg.msg1 = APPLE_RTKIT_EP_MGMT;

• ret = mbox_send(chan, &msg);

• ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret;

• ret = mbox_recv(chan, &msg, 100000);

• ret = mbox_recv(rtk->chan, &msg, 100000); if (ret < 0) return ret;

diff --git a/arch/arm/mach-apple/sart.c b/arch/arm/mach-apple/sart.c new file mode 100644 index 000000000000..2545b9ad0d14 --- /dev/null +++ b/arch/arm/mach-apple/sart.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: MIT +/*

• • The sart code is copied from m1n1 (https://github.com/AsahiLinux/m1n1) and
• • licensed as MIT.
• • (C) Copyright 2022 The Asahi Linux Contributors
• */

+#include <asm/io.h> +#include <asm/arch/sart.h>

+#include <linux/bitfield.h> +#include <linux/types.h>

+#include <malloc.h>

+#define APPLE_SART_MAX_ENTRIES 16

+/* This is probably a bitfield but the exact meaning of each bit is unknown. */ +#define APPLE_SART_FLAGS_ALLOW 0xff

+/* SARTv2 registers */ +#define APPLE_SART2_CONFIG(idx) (0x00 + 4 * (idx)) +#define APPLE_SART2_CONFIG_FLAGS GENMASK(31, 24) +#define APPLE_SART2_CONFIG_SIZE GENMASK(23, 0) +#define APPLE_SART2_CONFIG_SIZE_SHIFT 12 +#define APPLE_SART2_CONFIG_SIZE_MAX GENMASK(23, 0)

+#define APPLE_SART2_PADDR(idx) (0x40 + 4 * (idx)) +#define APPLE_SART2_PADDR_SHIFT 12

+/* SARTv3 registers */ +#define APPLE_SART3_CONFIG(idx) (0x00 + 4 * (idx))

+#define APPLE_SART3_PADDR(idx) (0x40 + 4 * (idx)) +#define APPLE_SART3_PADDR_SHIFT 12

+#define APPLE_SART3_SIZE(idx) (0x80 + 4 * (idx)) +#define APPLE_SART3_SIZE_SHIFT 12 +#define APPLE_SART3_SIZE_MAX GENMASK(29, 0)

+struct apple_sart {

• uintptr_t base;
• u32 protected_entries;
• void (*get_entry)(struct apple_sart *sart, int index, u8 *flags, void **paddr,

• 	  size_t *size);
  

• bool (*set_entry)(struct apple_sart *sart, int index, u8 flags, void *paddr,

• 	  size_t size);
  

+};

+static void sart2_get_entry(struct apple_sart *sart, int index, u8 *flags, void **paddr,

• 	    size_t *size)
  

+{

• u32 cfg = readl(sart->base + APPLE_SART2_CONFIG(index));
• *flags = FIELD_GET(APPLE_SART2_CONFIG_FLAGS, cfg);
• *size = (size_t)FIELD_GET(APPLE_SART2_CONFIG_SIZE, cfg) << APPLE_SART2_CONFIG_SIZE_SHIFT;
• *paddr = (void *)

• ((u64)readl(sart->base + APPLE_SART2_PADDR(index)) << APPLE_SART2_PADDR_SHIFT);
  

+}

+static bool sart2_set_entry(struct apple_sart *sart, int index, u8 flags, void *paddr_,

• 	    size_t size)
  

+{

• u32 cfg;
• u64 paddr = (u64)paddr_;
• if (size & ((1 << APPLE_SART2_CONFIG_SIZE_SHIFT) - 1))

• return false;
  

• if (paddr & ((1 << APPLE_SART2_PADDR_SHIFT) - 1))

• return false;
  

• size >>= APPLE_SART2_CONFIG_SIZE_SHIFT;
• paddr >>= APPLE_SART2_PADDR_SHIFT;
• if (size > APPLE_SART2_CONFIG_SIZE_MAX)

• return false;
  

• cfg = FIELD_PREP(APPLE_SART2_CONFIG_FLAGS, flags);
• cfg |= FIELD_PREP(APPLE_SART2_CONFIG_SIZE, size);
• writel(paddr, sart->base + APPLE_SART2_PADDR(index));
• writel(cfg, sart->base + APPLE_SART2_CONFIG(index));
• return true;

+}

+static void sart3_get_entry(struct apple_sart *sart, int index, u8 *flags, void **paddr,

• 	    size_t *size)
  

+{

• *flags = readl(sart->base + APPLE_SART3_CONFIG(index));
• *size = (size_t)readl(sart->base + APPLE_SART3_SIZE(index)) << APPLE_SART3_SIZE_SHIFT;
• *paddr = (void *)

• ((u64)readl(sart->base + APPLE_SART3_PADDR(index)) << APPLE_SART3_PADDR_SHIFT);
  

+}

+static bool sart3_set_entry(struct apple_sart *sart, int index, u8 flags, void *paddr_,

• 	    size_t size)
  

+{

• u64 paddr = (u64)paddr_;
• if (size & ((1 << APPLE_SART3_SIZE_SHIFT) - 1))

• return false;
  

• if (paddr & ((1 << APPLE_SART3_PADDR_SHIFT) - 1))

• return false;
  

• paddr >>= APPLE_SART3_PADDR_SHIFT;
• size >>= APPLE_SART3_SIZE_SHIFT;
• if (size > APPLE_SART3_SIZE_MAX)

• return false;
  

• writel(paddr, sart->base + APPLE_SART3_PADDR(index));
• writel(size, sart->base + APPLE_SART3_SIZE(index));
• writel(flags, sart->base + APPLE_SART3_CONFIG(index));
• return true;

+}

+struct apple_sart *sart_init(ofnode node) +{

• phys_addr_t base;
• u32 sart_version;
• struct apple_sart *sart;
• base = ofnode_get_addr(node);
• if (base == FDT_ADDR_T_NONE)

• return NULL;
  

• if (ofnode_device_is_compatible(node, "apple,sart2")) {

• sart_version = 2;
  

• } else if (ofnode_device_is_compatible(node, "apple,sart3")) {

• sart_version = 3;
  

• } else {

• printf("sart: unknown SART compatible: %sd\n",
  

•        ofnode_read_string(node, "compatible"));
  

• return NULL;
  

• }
• sart = calloc(sizeof(*sart), 1);
• if (!sart)

• return NULL;
  

• sart->base = base;
• switch (sart_version) {
• case 2:

• sart->get_entry = sart2_get_entry;
  

• sart->set_entry = sart2_set_entry;
  

• break;
  

• case 3:

• sart->get_entry = sart3_get_entry;
  

• sart->set_entry = sart3_set_entry;
  

• break;
  

• default:

• printf("sart: SART has unknown version %d\n", sart_version);
  

• free(sart);
  

• return NULL;
  

• }
• sart->protected_entries = 0;
• for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {

• void *paddr;
  

• u8 flags;
  

• size_t sz;
  

• sart->get_entry(sart, i, &flags, &paddr, &sz);
  

• if (flags)
  

• 	sart->protected_entries |= 1 << i;
  

• }
• return sart;

+}

+void sart_free(struct apple_sart *sart) +{

• for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {

• if (sart->protected_entries & (1 << i))
  

• 	continue;
  

• sart->set_entry(sart, i, 0, NULL, 0);
  

• }
• free(sart);

+}

+bool sart_add_allowed_region(struct apple_sart *sart, void *paddr, size_t sz) +{

• for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {

• void *e_paddr;
  

• u8 e_flags;
  

• size_t e_sz;
  

• if (sart->protected_entries & (1 << i))
  

• 	continue;
  

• sart->get_entry(sart, i, &e_flags, &e_paddr, &e_sz);
  

• if (e_flags)
  

• 	continue;
  

• return sart->set_entry(sart, i, APPLE_SART_FLAGS_ALLOW, paddr, sz);
  

• }
• printf("sart: no more free entries\n");
• return false;

+}

+bool sart_remove_allowed_region(struct apple_sart *sart, void *paddr, size_t sz) +{

• for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) {

• void *e_paddr;
  

• u8 e_flags;
  

• size_t e_sz;
  

• if (sart->protected_entries & (1 << i))
  

• 	continue;
  

• sart->get_entry(sart, i, &e_flags, &e_paddr, &e_sz);
  

• if (!e_flags)
  

• 	continue;
  

• if (e_paddr != paddr)
  

• 	continue;
  

• if (e_sz != sz)
  

• 	continue;
  

• return sart->set_entry(sart, i, 0, NULL, 0);
  

• }
• printf("sart: could not find entry to be removed\n");
• return false;

+} diff --git a/drivers/nvme/nvme_apple.c b/drivers/nvme/nvme_apple.c index d9d491c2be11..819b748dc02e 100644 --- a/drivers/nvme/nvme_apple.c +++ b/drivers/nvme/nvme_apple.c @@ -12,6 +12,7 @@

#include <asm/io.h> #include <asm/arch/rtkit.h> +#include <asm/arch/sart.h> #include <linux/iopoll.h>

/* ASC registers */ @@ -66,6 +67,8 @@ struct apple_nvme_priv { void *asc; /* ASC registers */ struct reset_ctl_bulk resets; /* ASC reset */ struct mbox_chan chan;

• struct apple_sart *sart;
• struct apple_rtkit *rtk; struct ans_nvmmu_tcb *tcbs[NVME_Q_NUM]; /* Submission queue TCBs */ u32 __iomem *q_db[NVME_Q_NUM]; /* Submission queue doorbell */

}; @@ -143,11 +146,51 @@ static void apple_nvme_complete_cmd(struct nvme_queue *nvmeq, nvmeq->sq_tail = tail; }

+static int nvme_shmem_setup(void *cookie, struct apple_rtkit_buffer *buf) +{

• struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;
• if (!buf || buf->dva || !buf->size)

• return -1;
  

• buf->buffer = memalign(SZ_16K, ALIGN(buf->size, SZ_16K));
• if (!buf->buffer)

• return -ENOMEM;
  

• if (!sart_add_allowed_region(priv->sart, buf->buffer, buf->size)) {

• free(buf->buffer);
  

• buf->buffer = NULL;
  

• buf->size = 0;
  

• return -1;
  

• }
• buf->dva = (u64)buf->buffer;
• return 0;

+}

+static void nvme_shmem_destroy(void *cookie, struct apple_rtkit_buffer *buf) +{

• struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;
• if (!buf)

• return;
  

• if (buf->buffer) {

• sart_remove_allowed_region(priv->sart, buf->buffer, buf->size);
  

• free(buf->buffer);
  

• buf->buffer = NULL;
  

• buf->size = 0;
  

• buf->dva = 0;
  

• }

+}

static int apple_nvme_probe(struct udevice *dev) { struct apple_nvme_priv *priv = dev_get_priv(dev); fdt_addr_t addr;

• u32 ctrl, stat;

• ofnode of_sart;

• u32 ctrl, stat, phandle; int ret;

  priv->base = dev_read_addr_ptr(dev);

@@ -167,12 +210,27 @@ static int apple_nvme_probe(struct udevice *dev) if (ret < 0) return ret;

• ret = dev_read_u32(dev, "apple,sart", &phandle);
• if (ret < 0)

• return ret;
  

• of_sart = ofnode_get_by_phandle(phandle);
• priv->sart = sart_init(of_sart);
• if (!priv->sart)

• return -EINVAL;
  

• ctrl = readl(priv->asc + REG_CPU_CTRL); writel(ctrl | REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);

• ret = apple_rtkit_init(&priv->chan);
• if (ret < 0)

• priv->rtk = apple_rtkit_init(&priv->chan, priv, nvme_shmem_setup, nvme_shmem_destroy);

• if (!priv->rtk)

• return -ENOMEM;
  

• ret = apple_rtkit_boot(priv->rtk);

• if (ret < 0) {

• printf("%s: NVMe apple_rtkit_boot returned: %d\n", __func__, ret);
  

  return ret;

• }

  ret = readl_poll_sleep_timeout(priv->base + ANS_BOOT_STATUS, stat, (stat == ANS_BOOT_STATUS_OK), 100,

@@ -206,11 +264,17 @@ static int apple_nvme_remove(struct udevice *dev)

nvme_shutdown(dev);

• apple_rtkit_shutdown(&priv->chan, APPLE_RTKIT_PWR_STATE_SLEEP);

• apple_rtkit_shutdown(priv->rtk, APPLE_RTKIT_PWR_STATE_SLEEP);

  ctrl = readl(priv->asc + REG_CPU_CTRL); writel(ctrl & ~REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);

• apple_rtkit_free(priv->rtk);

• priv->rtk = NULL;

• sart_free(priv->sart);

• priv->sart = NULL;

• reset_assert_bulk(&priv->resets); reset_deassert_bulk(&priv->resets);

-- 2.35.1