sqe.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: Copyright (c) 2022 Intel Corporation
 * SPDX-FileCopyrightText: Copyright (c) 2026 Infineon Technologies AG,
 * SPDX-FileCopyrightText: or an affiliate of Infineon Technologies AG. All rights reserved.
 *
 * SPDX-License-Identifier: Apache-2.0
 */

 
 
#ifndef ZEPHYR_INCLUDE_RTIO_SQE_H_
#define ZEPHYR_INCLUDE_RTIO_SQE_H_
 
#include <stdint.h>
#include <string.h>
#include <zephyr/kernel.h>
#include <zephyr/sys/atomic.h>
#include <zephyr/sys/mpsc_lockfree.h>
#include <zephyr/sys/util.h>
 
#ifdef __cplusplus
extern "C" {
#endif



#define RTIO_PRIO_LOW 0U

#define RTIO_PRIO_NORM 127U

#define RTIO_PRIO_HIGH 255U

 


#define RTIO_SQE_CHAINED BIT(0)

#define RTIO_SQE_TRANSACTION BIT(1)
 

#define RTIO_SQE_MEMPOOL_BUFFER BIT(2)

#define RTIO_SQE_CANCELED BIT(3) __DEPRECATED_MACRO

#define RTIO_SQE_MULTISHOT BIT(4)

#define RTIO_SQE_NO_RESPONSE BIT(5)

 


#define RTIO_OP_NOP 0

#define RTIO_OP_RX (RTIO_OP_NOP+1)

#define RTIO_OP_TX (RTIO_OP_RX+1)

#define RTIO_OP_TINY_TX (RTIO_OP_TX+1)

#define RTIO_OP_CALLBACK (RTIO_OP_TINY_TX+1)

#define RTIO_OP_TXRX (RTIO_OP_CALLBACK+1)

#define RTIO_OP_DELAY (RTIO_OP_TXRX+1)

#define RTIO_OP_I2C_RECOVER (RTIO_OP_DELAY+1)

#define RTIO_OP_I2C_CONFIGURE (RTIO_OP_I2C_RECOVER+1)

#define RTIO_OP_I3C_RECOVER (RTIO_OP_I2C_CONFIGURE+1)

#define RTIO_OP_I3C_CONFIGURE (RTIO_OP_I3C_RECOVER+1)

#define RTIO_OP_I3C_CCC (RTIO_OP_I3C_CONFIGURE+1)

#define RTIO_OP_AWAIT (RTIO_OP_I3C_CCC+1)

 


#define RTIO_IODEV_I2C_STOP BIT(1)

#define RTIO_IODEV_I2C_RESTART BIT(2)

#define RTIO_IODEV_I2C_10_BITS BIT(3)



#define RTIO_IODEV_I3C_STOP BIT(1)

#define RTIO_IODEV_I3C_RESTART BIT(2)

#define RTIO_IODEV_I3C_HDR BIT(3)

#define RTIO_IODEV_I3C_NBCH BIT(4)

#define RTIO_IODEV_I3C_NOACK_EXPECTED BIT(5)

#define RTIO_IODEV_I3C_HDR_MODE_MASK GENMASK(15, 8)

#define RTIO_IODEV_I3C_HDR_MODE_SET(flags) \
        FIELD_PREP(RTIO_IODEV_I3C_HDR_MODE_MASK, flags)

#define RTIO_IODEV_I3C_HDR_MODE_GET(flags) \
        FIELD_GET(RTIO_IODEV_I3C_HDR_MODE_MASK, flags)

#define RTIO_IODEV_I3C_HDR_CMD_CODE_MASK GENMASK(22, 16)

#define RTIO_IODEV_I3C_HDR_CMD_CODE_SET(flags) \
        FIELD_PREP(RTIO_IODEV_I3C_HDR_CMD_CODE_MASK, flags)

#define RTIO_IODEV_I3C_HDR_CMD_CODE_GET(flags) \
        FIELD_GET(RTIO_IODEV_I3C_HDR_CMD_CODE_MASK, flags)

 

struct rtio_sqe;
struct rtio_iodev_sqe;
struct rtio_iodev;
struct rtio;

typedef void (*rtio_callback_t)(struct rtio *r, const struct rtio_sqe *sqe, int res, void *arg0);

typedef void (*rtio_signaled_t)(struct rtio_iodev_sqe *iodev_sqe, void *userdata);

typedef uint32_t rtio_sqe_handle_t;

#define RTIO_SQE_HANDLE_INVALID ((rtio_sqe_handle_t)UINT32_MAX)

struct rtio_sqe {
        uint8_t op; 
 
        uint8_t prio; 
 
        uint16_t flags; 
 
        uint32_t iodev_flags; 
 
        const struct rtio_iodev *iodev; 

        void *userdata;

        union {

                struct {
                        uint32_t buf_len; 
                        const uint8_t *buf; 
                } tx;

                struct {
                        uint32_t buf_len; 
                        uint8_t *buf; 
                } rx;

                struct {
                        uint8_t buf_len; 
                        uint8_t buf[7]; 
                } tiny_tx;

                struct {
                        rtio_callback_t callback; 
                        void *arg0; 
                } callback;

                struct {
                        uint32_t buf_len; 
                        const uint8_t *tx_buf; 
                        uint8_t *rx_buf; 
                } txrx;
 
#ifdef CONFIG_RTIO_OP_DELAY
                struct {
                        k_timeout_t timeout; 
                } delay;
#endif

                uint32_t i2c_config;

                struct {
                        /* enum i3c_config_type type; */
                        int type; 
                        void *config; 
                } i3c_config;

                /* struct i3c_ccc_payload *ccc_payload; */
                void *ccc_payload;
        };
};
 

#define RTIO_SQE_GEN_MASK GENMASK(15, 0)
#define RTIO_SQE_ALLOCD   BIT(16)
#define RTIO_SQE_STATUS_CANCELED BIT(17)

static inline uint16_t rtio_sqe_handle_index(rtio_sqe_handle_t handle)
{
        return (uint16_t)(handle >> 16);
}

static inline uint16_t rtio_sqe_handle_generation(rtio_sqe_handle_t handle)
{
        return (uint16_t)(handle & RTIO_SQE_GEN_MASK);
}

struct rtio_iodev_sqe {
        struct rtio_sqe sqe;
        struct mpsc_node q;
        struct rtio_iodev_sqe *next;
        struct rtio *r;

        atomic_t status;

        union {
                struct {
                        uint8_t *buf;     
                        uint32_t buf_len; 
                } rx_bind;

                struct {
                        atomic_t ok;              
                        rtio_signaled_t callback; 
                        void *userdata;           
                } await;
 
#ifdef CONFIG_RTIO_OP_DELAY
                k_timepoint_t delay_expiry;
#endif
        } rt;
};
 

/* Ensure the rtio_iodev_sqe never grows beyond a common cacheline size of 64 bytes */
#if CONFIG_RTIO_SQE_CACHELINE_CHECK
#ifdef CONFIG_DCACHE_LINE_SIZE
#define RTIO_CACHE_LINE_SIZE CONFIG_DCACHE_LINE_SIZE
#else
#define RTIO_CACHE_LINE_SIZE 64
#endif
BUILD_ASSERT(sizeof(struct rtio_iodev_sqe) <= RTIO_CACHE_LINE_SIZE,
        "RTIO performs best when the submissions queue entries are less than a cache line");
#endif

static inline void rtio_sqe_prep_nop(struct rtio_sqe *sqe,
                                const struct rtio_iodev *iodev,
                                void *userdata)
{
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_NOP;
        sqe->iodev = iodev;
        sqe->userdata = userdata;
}
 

static inline void _rtio_sqe_prep_read(struct rtio_sqe *sqe,
                                      const struct rtio_iodev *iodev,
                                      int8_t prio,
                                      uint8_t *buf,
                                      uint32_t len,
                                      void *userdata)
{
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_RX;
        sqe->prio = prio;
        sqe->iodev = iodev;
        sqe->rx.buf_len = len;
        sqe->rx.buf = buf;
        sqe->userdata = userdata;
}

static inline void rtio_sqe_prep_read(struct rtio_sqe *sqe,
                                      const struct rtio_iodev *iodev,
                                      int8_t prio,
                                      uint8_t *buf,
                                      uint32_t len,
                                      void *userdata)
{
        __ASSERT_NO_MSG(len > 0 && buf != NULL);
 
        _rtio_sqe_prep_read(sqe, iodev, prio, buf, len, userdata);
}

static inline void rtio_sqe_prep_read_with_pool(struct rtio_sqe *sqe,
                                                const struct rtio_iodev *iodev, int8_t prio,
                                                void *userdata)
{
        _rtio_sqe_prep_read(sqe, iodev, prio, NULL, 0, userdata);
        sqe->flags = RTIO_SQE_MEMPOOL_BUFFER;
}

static inline void rtio_sqe_prep_read_multishot(struct rtio_sqe *sqe,
                                                const struct rtio_iodev *iodev, int8_t prio,
                                                void *userdata)
{
        rtio_sqe_prep_read_with_pool(sqe, iodev, prio, userdata);
        sqe->flags |= RTIO_SQE_MULTISHOT;
}

static inline void rtio_sqe_prep_write(struct rtio_sqe *sqe,
                                       const struct rtio_iodev *iodev,
                                       int8_t prio,
                                       const uint8_t *buf,
                                       uint32_t len,
                                       void *userdata)
{
        __ASSERT_NO_MSG(len > 0 && buf != NULL);
 
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_TX;
        sqe->prio = prio;
        sqe->iodev = iodev;
        sqe->tx.buf_len = len;
        sqe->tx.buf = buf;
        sqe->userdata = userdata;
}

static inline void rtio_sqe_prep_tiny_write(struct rtio_sqe *sqe,
                                            const struct rtio_iodev *iodev,
                                            int8_t prio,
                                            const uint8_t *tiny_write_data,
                                            uint8_t tiny_write_len,
                                            void *userdata)
{
        __ASSERT_NO_MSG(tiny_write_len <= sizeof(sqe->tiny_tx.buf));
        __ASSERT_NO_MSG(tiny_write_len > 0 && tiny_write_data != NULL);
 
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_TINY_TX;
        sqe->prio = prio;
        sqe->iodev = iodev;
        sqe->tiny_tx.buf_len = tiny_write_len;
        memcpy(sqe->tiny_tx.buf, tiny_write_data, tiny_write_len);
        sqe->userdata = userdata;
}

static inline void rtio_sqe_prep_callback(struct rtio_sqe *sqe,
                                          rtio_callback_t callback,
                                          void *arg0,
                                          void *userdata)
{
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_CALLBACK;
        sqe->prio = 0;
        sqe->iodev = NULL;
        sqe->callback.callback = callback;
        sqe->callback.arg0 = arg0;
        sqe->userdata = userdata;
}

static inline void rtio_sqe_prep_callback_no_cqe(struct rtio_sqe *sqe,
                                                 rtio_callback_t callback,
                                                 void *arg0,
                                                 void *userdata)
{
        rtio_sqe_prep_callback(sqe, callback, arg0, userdata);
        sqe->flags |= RTIO_SQE_NO_RESPONSE;
}

static inline void rtio_sqe_prep_transceive(struct rtio_sqe *sqe,
                                            const struct rtio_iodev *iodev,
                                            int8_t prio,
                                            const uint8_t *tx_buf,
                                            uint8_t *rx_buf,
                                            uint32_t buf_len,
                                            void *userdata)
{
        __ASSERT_NO_MSG(buf_len > 0 && tx_buf != NULL && rx_buf != NULL);
 
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_TXRX;
        sqe->prio = prio;
        sqe->iodev = iodev;
        sqe->txrx.buf_len = buf_len;
        sqe->txrx.tx_buf = tx_buf;
        sqe->txrx.rx_buf = rx_buf;
        sqe->userdata = userdata;
}

static inline void rtio_sqe_prep_await(struct rtio_sqe *sqe,
                                       const struct rtio_iodev *iodev,
                                       int8_t prio,
                                       void *userdata)
{
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_AWAIT;
        sqe->prio = prio;
        sqe->iodev = iodev;
        sqe->userdata = userdata;
}

static inline void rtio_sqe_prep_await_iodev(struct rtio_sqe *sqe, const struct rtio_iodev *iodev,
                                             int8_t prio, void *userdata)
{
        __ASSERT_NO_MSG(iodev != NULL);
        rtio_sqe_prep_await(sqe, iodev, prio, userdata);
}

static inline void rtio_sqe_prep_await_executor(struct rtio_sqe *sqe, int8_t prio, void *userdata)
{
        rtio_sqe_prep_await(sqe, NULL, prio, userdata);
}

#ifdef CONFIG_RTIO_OP_DELAY
extern struct rtio_iodev rtio_timeout_iodev;
 
static inline void rtio_sqe_prep_delay(struct rtio_sqe *sqe,
                                       k_timeout_t timeout,
                                       void *userdata)
{
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_DELAY;
        sqe->prio = 0;
        sqe->iodev = &rtio_timeout_iodev;
        sqe->delay.timeout = timeout;
        sqe->userdata = userdata;
}
#else
#define rtio_sqe_prep_delay(sqe, timeout, userdata) \
        BUILD_ASSERT(false, "CONFIG_RTIO_OP_DELAY not enabled")
#endif

static inline void rtio_sqe_prep_i2c_configure(struct rtio_sqe *sqe,
                                               const struct rtio_iodev *iodev,
                                               int8_t prio,
                                               uint32_t dev_config,
                                               void *userdata)
{
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_I2C_CONFIGURE;
        sqe->prio = prio;
        sqe->iodev = iodev;
        sqe->i2c_config = dev_config;
        sqe->userdata = userdata;
}

static inline void rtio_sqe_prep_i2c_recover(struct rtio_sqe *sqe,
                                             const struct rtio_iodev *iodev,
                                             int8_t prio,
                                             void *userdata)
{
        memset(sqe, 0, sizeof(struct rtio_sqe));
        sqe->op = RTIO_OP_I2C_RECOVER;
        sqe->prio = prio;
        sqe->iodev = iodev;
        sqe->userdata = userdata;
}

static inline struct rtio_iodev_sqe *rtio_txn_next(const struct rtio_iodev_sqe *iodev_sqe)
{
        struct rtio_iodev_sqe *next = NULL;
 
        SYS_PORT_TRACING_FUNC_ENTER(rtio, txn_next, iodev_sqe->r, iodev_sqe);
        if (iodev_sqe->sqe.flags & RTIO_SQE_TRANSACTION) {
                next = iodev_sqe->next;
        }
        SYS_PORT_TRACING_FUNC_EXIT(rtio, txn_next, iodev_sqe->r, next);
        return next;
}
 

static inline struct rtio_iodev_sqe *rtio_chain_next(const struct rtio_iodev_sqe *iodev_sqe)
{
        struct rtio_iodev_sqe *next = NULL;
 
        SYS_PORT_TRACING_FUNC_ENTER(rtio, txn_next, iodev_sqe->r, iodev_sqe);
        if (iodev_sqe->sqe.flags & RTIO_SQE_CHAINED) {
                next = iodev_sqe->next;
        }
        SYS_PORT_TRACING_FUNC_EXIT(rtio, txn_next, iodev_sqe->r, next);
        return next;
}

static inline struct rtio_iodev_sqe *rtio_iodev_sqe_next(const struct rtio_iodev_sqe *iodev_sqe)
{
        return iodev_sqe->next;
}

static inline bool rtio_iodev_sqe_is_canceled(const struct rtio_iodev_sqe *iodev_sqe)
{
        return (atomic_get(&iodev_sqe->status) & RTIO_SQE_STATUS_CANCELED) != 0;
}

static inline void rtio_iodev_sqe_set_canceled(struct rtio_iodev_sqe *iodev_sqe)
{
        atomic_or(&iodev_sqe->status, RTIO_SQE_STATUS_CANCELED);
}

static inline void rtio_iodev_sqe_await_signal(struct rtio_iodev_sqe *iodev_sqe,
                                               rtio_signaled_t callback,
                                               void *userdata)
{
        iodev_sqe->rt.await.callback = callback;
        iodev_sqe->rt.await.userdata = userdata;
 
        if (!atomic_cas(&iodev_sqe->rt.await.ok, 0, 1)) {
                callback(iodev_sqe, userdata);
        }
}
 
/* Private structures and functions used for the pool of sqe structures */
 
struct rtio_sqe_pool {
        struct mpsc free_q;
        const uint16_t pool_size;
        uint16_t pool_free;
        struct rtio_iodev_sqe *pool;
};
 
static inline struct rtio_iodev_sqe *rtio_sqe_pool_alloc(struct rtio_sqe_pool *pool)
{
        struct mpsc_node *node = mpsc_pop(&pool->free_q);
 
        if (node == NULL) {
                return NULL;
        }
 
        struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
 
        /* Reset the runtime scratch so per-op state (await ok, mempool binding)
         * starts clean for this allocation.
         */
        memset(&iodev_sqe->rt, 0, sizeof(iodev_sqe->rt));
 
        /* The chain link is only valid once the executor links the entry at
         * submit time; clear it so a cancel before then does not walk the
         * previous occupant's chain.
         */
        iodev_sqe->next = NULL;
 
        /* Mark the slot live. The generation persists from the previous free so a
         * handle captured against the prior occupant no longer matches.
         */
        atomic_or(&iodev_sqe->status, RTIO_SQE_ALLOCD);
 
        pool->pool_free--;
 
        return iodev_sqe;
}
 
static inline void rtio_sqe_pool_free(struct rtio_sqe_pool *pool, struct rtio_iodev_sqe *iodev_sqe)
{
        /* Bump the generation and clear ALLOCD so any outstanding handle to this
         * slot is rejected by rtio_iodev_sqe_from_handle() once it is recycled.
         */
        atomic_val_t gen = (atomic_get(&iodev_sqe->status) + 1) & RTIO_SQE_GEN_MASK;
 
        atomic_set(&iodev_sqe->status, gen);
 
        mpsc_push(&pool->free_q, &iodev_sqe->q);
 
        pool->pool_free++;
}
 
#if CONFIG_RTIO_SQE_PLACEMENT_DTCM
#define RTIO_SQE_MEM Z_GENERIC_SECTION(".dtcm_bss") static
#elif defined(CONFIG_RTIO_SQE_PLACEMENT_NOCACHE)
#define RTIO_SQE_MEM __nocache static
#else
#define RTIO_SQE_MEM static
#endif
 
/* Do not try and reformat the macros */
/* clang-format off */
 
#define Z_RTIO_SQE_POOL_DEFINE(name, sz)                        \
        RTIO_SQE_MEM struct rtio_iodev_sqe CONCAT(_sqe_pool_, name)[sz];        \
        STRUCT_SECTION_ITERABLE(rtio_sqe_pool, name) = {        \
                .free_q = MPSC_INIT((name.free_q)),     \
                .pool_size = sz,                                \
                .pool_free = sz,                                \
                .pool = CONCAT(_sqe_pool_, name),               \
        }
 
/* clang-format on */

 
#ifdef __cplusplus
}
#endif
 
#endif /* ZEPHYR_INCLUDE_RTIO_SQE_H_ */