drm/radeon: Implement SDMA interface functions

This patch implements the new SDMA interface functions. It also adds defines
and structures related to SDMA registers.

v2: Removed init_sdma_engines() from interface. Initialization is done in
radeon.

v3:
- Removed unused defines.
- Added SDMA_ prefix to defines that didn't have them.

Signed-off-by: Ben Goz <ben.goz@amd.com>
Signed-off-by: Oded Gabbay <oded.gabbay@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/radeon/cik_reg.h b/drivers/gpu/drm/radeon/cik_reg.h
index 79c45e8..bbb8f2e 100644 (file)
--- a/drivers/gpu/drm/radeon/cik_reg.h
+++ b/drivers/gpu/drm/radeon/cik_reg.h
@@ -147,10 +147,42 @@
 
 #define CIK_LB_DESKTOP_HEIGHT                     0x6b0c
 
+#define KFD_CIK_SDMA_QUEUE_OFFSET              0x200
+
 #define CP_HQD_IQ_RPTR                                 0xC970u
 #define AQL_ENABLE                                     (1U << 0)
-
-#define IDLE                                   (1 << 2)
+#define SDMA0_RLC0_RB_CNTL                             0xD400u
+#define        SDMA_RB_VMID(x)                                 (x << 24)
+#define        SDMA0_RLC0_RB_BASE                              0xD404u
+#define        SDMA0_RLC0_RB_BASE_HI                           0xD408u
+#define        SDMA0_RLC0_RB_RPTR                              0xD40Cu
+#define        SDMA0_RLC0_RB_WPTR                              0xD410u
+#define        SDMA0_RLC0_RB_WPTR_POLL_CNTL                    0xD414u
+#define        SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI                 0xD418u
+#define        SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO                 0xD41Cu
+#define        SDMA0_RLC0_RB_RPTR_ADDR_HI                      0xD420u
+#define        SDMA0_RLC0_RB_RPTR_ADDR_LO                      0xD424u
+#define        SDMA0_RLC0_IB_CNTL                              0xD428u
+#define        SDMA0_RLC0_IB_RPTR                              0xD42Cu
+#define        SDMA0_RLC0_IB_OFFSET                            0xD430u
+#define        SDMA0_RLC0_IB_BASE_LO                           0xD434u
+#define        SDMA0_RLC0_IB_BASE_HI                           0xD438u
+#define        SDMA0_RLC0_IB_SIZE                              0xD43Cu
+#define        SDMA0_RLC0_SKIP_CNTL                            0xD440u
+#define        SDMA0_RLC0_CONTEXT_STATUS                       0xD444u
+#define        SDMA_RLC_IDLE                                   (1 << 2)
+#define        SDMA0_RLC0_DOORBELL                             0xD448u
+#define        SDMA_OFFSET(x)                                  (x << 0)
+#define        SDMA_DB_ENABLE                                  (1 << 28)
+#define        SDMA0_RLC0_VIRTUAL_ADDR                         0xD49Cu
+#define        SDMA_ATC                                        (1 << 0)
+#define        SDMA_VA_PTR32                                   (1 << 4)
+#define        SDMA_VA_SHARED_BASE(x)                          (x << 8)
+#define        SDMA0_RLC0_APE1_CNTL                            0xD4A0u
+#define        SDMA0_RLC0_DOORBELL_LOG                         0xD4A4u
+#define        SDMA0_RLC0_WATERMARK                            0xD4A8u
+#define        SDMA0_CNTL                                      0xD010
+#define        SDMA1_CNTL                                      0xD810
 
 struct cik_mqd {
        uint32_t header;
@@ -283,4 +315,137 @@ struct cik_mqd {
        uint32_t queue_doorbell_id15;
 };
 
+struct cik_sdma_rlc_registers {
+       uint32_t sdma_rlc_rb_cntl;
+       uint32_t sdma_rlc_rb_base;
+       uint32_t sdma_rlc_rb_base_hi;
+       uint32_t sdma_rlc_rb_rptr;
+       uint32_t sdma_rlc_rb_wptr;
+       uint32_t sdma_rlc_rb_wptr_poll_cntl;
+       uint32_t sdma_rlc_rb_wptr_poll_addr_hi;
+       uint32_t sdma_rlc_rb_wptr_poll_addr_lo;
+       uint32_t sdma_rlc_rb_rptr_addr_hi;
+       uint32_t sdma_rlc_rb_rptr_addr_lo;
+       uint32_t sdma_rlc_ib_cntl;
+       uint32_t sdma_rlc_ib_rptr;
+       uint32_t sdma_rlc_ib_offset;
+       uint32_t sdma_rlc_ib_base_lo;
+       uint32_t sdma_rlc_ib_base_hi;
+       uint32_t sdma_rlc_ib_size;
+       uint32_t sdma_rlc_skip_cntl;
+       uint32_t sdma_rlc_context_status;
+       uint32_t sdma_rlc_doorbell;
+       uint32_t sdma_rlc_virtual_addr;
+       uint32_t sdma_rlc_ape1_cntl;
+       uint32_t sdma_rlc_doorbell_log;
+       uint32_t reserved_22;
+       uint32_t reserved_23;
+       uint32_t reserved_24;
+       uint32_t reserved_25;
+       uint32_t reserved_26;
+       uint32_t reserved_27;
+       uint32_t reserved_28;
+       uint32_t reserved_29;
+       uint32_t reserved_30;
+       uint32_t reserved_31;
+       uint32_t reserved_32;
+       uint32_t reserved_33;
+       uint32_t reserved_34;
+       uint32_t reserved_35;
+       uint32_t reserved_36;
+       uint32_t reserved_37;
+       uint32_t reserved_38;
+       uint32_t reserved_39;
+       uint32_t reserved_40;
+       uint32_t reserved_41;
+       uint32_t reserved_42;
+       uint32_t reserved_43;
+       uint32_t reserved_44;
+       uint32_t reserved_45;
+       uint32_t reserved_46;
+       uint32_t reserved_47;
+       uint32_t reserved_48;
+       uint32_t reserved_49;
+       uint32_t reserved_50;
+       uint32_t reserved_51;
+       uint32_t reserved_52;
+       uint32_t reserved_53;
+       uint32_t reserved_54;
+       uint32_t reserved_55;
+       uint32_t reserved_56;
+       uint32_t reserved_57;
+       uint32_t reserved_58;
+       uint32_t reserved_59;
+       uint32_t reserved_60;
+       uint32_t reserved_61;
+       uint32_t reserved_62;
+       uint32_t reserved_63;
+       uint32_t reserved_64;
+       uint32_t reserved_65;
+       uint32_t reserved_66;
+       uint32_t reserved_67;
+       uint32_t reserved_68;
+       uint32_t reserved_69;
+       uint32_t reserved_70;
+       uint32_t reserved_71;
+       uint32_t reserved_72;
+       uint32_t reserved_73;
+       uint32_t reserved_74;
+       uint32_t reserved_75;
+       uint32_t reserved_76;
+       uint32_t reserved_77;
+       uint32_t reserved_78;
+       uint32_t reserved_79;
+       uint32_t reserved_80;
+       uint32_t reserved_81;
+       uint32_t reserved_82;
+       uint32_t reserved_83;
+       uint32_t reserved_84;
+       uint32_t reserved_85;
+       uint32_t reserved_86;
+       uint32_t reserved_87;
+       uint32_t reserved_88;
+       uint32_t reserved_89;
+       uint32_t reserved_90;
+       uint32_t reserved_91;
+       uint32_t reserved_92;
+       uint32_t reserved_93;
+       uint32_t reserved_94;
+       uint32_t reserved_95;
+       uint32_t reserved_96;
+       uint32_t reserved_97;
+       uint32_t reserved_98;
+       uint32_t reserved_99;
+       uint32_t reserved_100;
+       uint32_t reserved_101;
+       uint32_t reserved_102;
+       uint32_t reserved_103;
+       uint32_t reserved_104;
+       uint32_t reserved_105;
+       uint32_t reserved_106;
+       uint32_t reserved_107;
+       uint32_t reserved_108;
+       uint32_t reserved_109;
+       uint32_t reserved_110;
+       uint32_t reserved_111;
+       uint32_t reserved_112;
+       uint32_t reserved_113;
+       uint32_t reserved_114;
+       uint32_t reserved_115;
+       uint32_t reserved_116;
+       uint32_t reserved_117;
+       uint32_t reserved_118;
+       uint32_t reserved_119;
+       uint32_t reserved_120;
+       uint32_t reserved_121;
+       uint32_t reserved_122;
+       uint32_t reserved_123;
+       uint32_t reserved_124;
+       uint32_t reserved_125;
+       uint32_t reserved_126;
+       uint32_t reserved_127;
+       uint32_t sdma_engine_id;
+       uint32_t sdma_queue_id;
+};
+
 #endif

diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c b/drivers/gpu/drm/radeon/radeon_kfd.c
index 242fd8b..0291681 100644 (file)
--- a/drivers/gpu/drm/radeon/radeon_kfd.c
+++ b/drivers/gpu/drm/radeon/radeon_kfd.c
@@ -71,13 +71,16 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
 
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
                        uint32_t queue_id, uint32_t __user *wptr);
-
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
 static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
                                uint32_t pipe_id, uint32_t queue_id);
 
 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
                                unsigned int timeout, uint32_t pipe_id,
                                uint32_t queue_id);
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+                               unsigned int timeout);
 
 static const struct kfd2kgd_calls kfd2kgd = {
        .init_sa_manager = init_sa_manager,
@@ -92,8 +95,11 @@ static const struct kfd2kgd_calls kfd2kgd = {
        .init_memory = kgd_init_memory,
        .init_pipeline = kgd_init_pipeline,
        .hqd_load = kgd_hqd_load,
+       .hqd_sdma_load = kgd_hqd_sdma_load,
        .hqd_is_occupies = kgd_hqd_is_occupies,
+       .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
        .hqd_destroy = kgd_hqd_destroy,
+       .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
        .get_fw_version = get_fw_version
 };
 
@@ -435,11 +441,28 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
        return 0;
 }
 
+static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
+{
+       uint32_t retval;
+
+       retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
+                       m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
+
+       pr_debug("kfd: sdma base address: 0x%x\n", retval);
+
+       return retval;
+}
+
 static inline struct cik_mqd *get_mqd(void *mqd)
 {
        return (struct cik_mqd *)mqd;
 }
 
+static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+{
+       return (struct cik_sdma_rlc_registers *)mqd;
+}
+
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
                        uint32_t queue_id, uint32_t __user *wptr)
 {
@@ -517,6 +540,45 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
        return 0;
 }
 
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+{
+       struct cik_sdma_rlc_registers *m;
+       uint32_t sdma_base_addr;
+
+       m = get_sdma_mqd(mqd);
+       sdma_base_addr = get_sdma_base_addr(m);
+
+       write_register(kgd,
+                       sdma_base_addr + SDMA0_RLC0_VIRTUAL_ADDR,
+                       m->sdma_rlc_virtual_addr);
+
+       write_register(kgd,
+                       sdma_base_addr + SDMA0_RLC0_RB_BASE,
+                       m->sdma_rlc_rb_base);
+
+       write_register(kgd,
+                       sdma_base_addr + SDMA0_RLC0_RB_BASE_HI,
+                       m->sdma_rlc_rb_base_hi);
+
+       write_register(kgd,
+                       sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_LO,
+                       m->sdma_rlc_rb_rptr_addr_lo);
+
+       write_register(kgd,
+                       sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_HI,
+                       m->sdma_rlc_rb_rptr_addr_hi);
+
+       write_register(kgd,
+                       sdma_base_addr + SDMA0_RLC0_DOORBELL,
+                       m->sdma_rlc_doorbell);
+
+       write_register(kgd,
+                       sdma_base_addr + SDMA0_RLC0_RB_CNTL,
+                       m->sdma_rlc_rb_cntl);
+
+       return 0;
+}
+
 static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
                                uint32_t pipe_id, uint32_t queue_id)
 {
@@ -538,6 +600,24 @@ static bool kgd_hqd_is_occupies(struct kgd_dev *kgd, uint64_t queue_address,
        return retval;
 }
 
+static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
+{
+       struct cik_sdma_rlc_registers *m;
+       uint32_t sdma_base_addr;
+       uint32_t sdma_rlc_rb_cntl;
+
+       m = get_sdma_mqd(mqd);
+       sdma_base_addr = get_sdma_base_addr(m);
+
+       sdma_rlc_rb_cntl = read_register(kgd,
+                                       sdma_base_addr + SDMA0_RLC0_RB_CNTL);
+
+       if (sdma_rlc_rb_cntl & SDMA_RB_ENABLE)
+               return true;
+
+       return false;
+}
+
 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
                                unsigned int timeout, uint32_t pipe_id,
                                uint32_t queue_id)
@@ -566,6 +646,39 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
        return 0;
 }
 
+static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
+                               unsigned int timeout)
+{
+       struct cik_sdma_rlc_registers *m;
+       uint32_t sdma_base_addr;
+       uint32_t temp;
+
+       m = get_sdma_mqd(mqd);
+       sdma_base_addr = get_sdma_base_addr(m);
+
+       temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL);
+       temp = temp & ~SDMA_RB_ENABLE;
+       write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp);
+
+       while (true) {
+               temp = read_register(kgd, sdma_base_addr +
+                                               SDMA0_RLC0_CONTEXT_STATUS);
+               if (temp & SDMA_RLC_IDLE)
+                       break;
+               if (timeout == 0)
+                       return -ETIME;
+               msleep(20);
+               timeout -= 20;
+       }
+
+       write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0);
+       write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0);
+       write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0);
+       write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0);
+
+       return 0;
+}
+
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 {
        struct radeon_device *rdev = (struct radeon_device *) kgd;