drivers/scsi/ufs/ufshcd.c

   1 /*
   2  * Universal Flash Storage Host controller driver Core
   3  *
   4  * This code is based on drivers/scsi/ufs/ufshcd.c
   5  * Copyright (C) 2011-2013 Samsung India Software Operations
   6  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   7  *
   8  * Authors:
   9  *      Santosh Yaraganavi <santosh.sy@samsung.com>
  10  *      Vinayak Holikatti <h.vinayak@samsung.com>
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  * See the COPYING file in the top-level directory or visit
  17  * <http://www.gnu.org/licenses/gpl-2.0.html>
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  *
  24  * This program is provided "AS IS" and "WITH ALL FAULTS" and
  25  * without warranty of any kind. You are solely responsible for
  26  * determining the appropriateness of using and distributing
  27  * the program and assume all risks associated with your exercise
  28  * of rights with respect to the program, including but not limited
  29  * to infringement of third party rights, the risks and costs of
  30  * program errors, damage to or loss of data, programs or equipment,
  31  * and unavailability or interruption of operations. Under no
  32  * circumstances will the contributor of this Program be liable for
  33  * any damages of any kind arising from your use or distribution of
  34  * this program.
  35  *
  36  * The Linux Foundation chooses to take subject only to the GPLv2
  37  * license terms, and distributes only under these terms.
  38  */
  39
  40 #include <linux/async.h>
  41 #include <linux/devfreq.h>
  42 #include <linux/nls.h>
  43 #include <linux/of.h>
  44 #include "ufshcd.h"
  45 #include "ufs_quirks.h"
  46 #include "unipro.h"
  47
  48 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  49                                  UTP_TASK_REQ_COMPL |\
  50                                  UFSHCD_ERROR_MASK)
  51 /* UIC command timeout, unit: ms */
  52 #define UIC_CMD_TIMEOUT 500
  53
  54 /* NOP OUT retries waiting for NOP IN response */
  55 #define NOP_OUT_RETRIES    10
  56 /* Timeout after 30 msecs if NOP OUT hangs without response */
  57 #define NOP_OUT_TIMEOUT    30 /* msecs */
  58
  59 /* Query request retries */
  60 #define QUERY_REQ_RETRIES 10
  61 /* Query request timeout */
  62 #define QUERY_REQ_TIMEOUT 30 /* msec */
  63 /*
  64  * Query request timeout for fDeviceInit flag
  65  * fDeviceInit query response time for some devices is too large that default
  66  * QUERY_REQ_TIMEOUT may not be enough for such devices.
  67  */
  68 #define QUERY_FDEVICEINIT_REQ_TIMEOUT 600 /* msec */
  69
  70 /* Task management command timeout */
  71 #define TM_CMD_TIMEOUT  100 /* msecs */
  72
  73 /* maximum number of retries for a general UIC command  */
  74 #define UFS_UIC_COMMAND_RETRIES 3
  75
  76 /* maximum number of link-startup retries */
  77 #define DME_LINKSTARTUP_RETRIES 3
  78
  79 /* Maximum retries for Hibern8 enter */
  80 #define UIC_HIBERN8_ENTER_RETRIES 3
  81
  82 /* maximum number of reset retries before giving up */
  83 #define MAX_HOST_RESET_RETRIES 5
  84
  85 /* Expose the flag value from utp_upiu_query.value */
  86 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  87
  88 /* Interrupt aggregation default timeout, unit: 40us */
  89 #define INT_AGGR_DEF_TO 0x02
  90
  91 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
  92         ({                                                              \
  93                 int _ret;                                               \
  94                 if (_on)                                                \
  95                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
  96                 else                                                    \
  97                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
  98                 _ret;                                                   \
  99         })
 100
 101 static u32 ufs_query_desc_max_size[] = {
 102         QUERY_DESC_DEVICE_MAX_SIZE,
 103         QUERY_DESC_CONFIGURAION_MAX_SIZE,
 104         QUERY_DESC_UNIT_MAX_SIZE,
 105         QUERY_DESC_RFU_MAX_SIZE,
 106         QUERY_DESC_INTERCONNECT_MAX_SIZE,
 107         QUERY_DESC_STRING_MAX_SIZE,
 108         QUERY_DESC_RFU_MAX_SIZE,
 109         QUERY_DESC_GEOMETRY_MAX_SIZE,
 110         QUERY_DESC_POWER_MAX_SIZE,
 111         QUERY_DESC_RFU_MAX_SIZE,
 112 };
 113
 114 enum {
 115         UFSHCD_MAX_CHANNEL      = 0,
 116         UFSHCD_MAX_ID           = 1,
 117         UFSHCD_CMD_PER_LUN      = 32,
 118         UFSHCD_CAN_QUEUE        = 32,
 119 };
 120
 121 /* UFSHCD states */
 122 enum {
 123         UFSHCD_STATE_RESET,
 124         UFSHCD_STATE_ERROR,
 125         UFSHCD_STATE_OPERATIONAL,
 126 };
 127
 128 /* UFSHCD error handling flags */
 129 enum {
 130         UFSHCD_EH_IN_PROGRESS = (1 << 0),
 131 };
 132
 133 /* UFSHCD UIC layer error flags */
 134 enum {
 135         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 136         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 137         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 138         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 139         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 140         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 141 };
 142
 143 /* Interrupt configuration options */
 144 enum {
 145         UFSHCD_INT_DISABLE,
 146         UFSHCD_INT_ENABLE,
 147         UFSHCD_INT_CLEAR,
 148 };
 149
 150 #define ufshcd_set_eh_in_progress(h) \
 151         (h->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 152 #define ufshcd_eh_in_progress(h) \
 153         (h->eh_flags & UFSHCD_EH_IN_PROGRESS)
 154 #define ufshcd_clear_eh_in_progress(h) \
 155         (h->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 156
 157 #define ufshcd_set_ufs_dev_active(h) \
 158         ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
 159 #define ufshcd_set_ufs_dev_sleep(h) \
 160         ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
 161 #define ufshcd_set_ufs_dev_poweroff(h) \
 162         ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
 163 #define ufshcd_is_ufs_dev_active(h) \
 164         ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
 165 #define ufshcd_is_ufs_dev_sleep(h) \
 166         ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
 167 #define ufshcd_is_ufs_dev_poweroff(h) \
 168         ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
 169
 170 static struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 171         {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 172         {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 173         {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 174         {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 175         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 176         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 177 };
 178
 179 static inline enum ufs_dev_pwr_mode
 180 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 181 {
 182         return ufs_pm_lvl_states[lvl].dev_state;
 183 }
 184
 185 static inline enum uic_link_state
 186 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 187 {
 188         return ufs_pm_lvl_states[lvl].link_state;
 189 }
 190
 191 static void ufshcd_tmc_handler(struct ufs_hba *hba);
 192 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 193 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 194 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 195 static void ufshcd_hba_exit(struct ufs_hba *hba);
 196 static int ufshcd_probe_hba(struct ufs_hba *hba);
 197 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
 198                                  bool skip_ref_clk);
 199 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 200 static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused);
 201 static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
 202 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
 203 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 204 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 205 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 206 static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
 207                 struct ufs_pa_layer_attr *desired_pwr_mode);
 208 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 209                              struct ufs_pa_layer_attr *pwr_mode);
 210 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
 211 {
 212         return tag >= 0 && tag < hba->nutrs;
 213 }
 214
 215 static inline int ufshcd_enable_irq(struct ufs_hba *hba)
 216 {
 217         int ret = 0;
 218
 219         if (!hba->is_irq_enabled) {
 220                 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
 221                                 hba);
 222                 if (ret)
 223                         dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
 224                                 __func__, ret);
 225                 hba->is_irq_enabled = true;
 226         }
 227
 228         return ret;
 229 }
 230
 231 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 232 {
 233         if (hba->is_irq_enabled) {
 234                 free_irq(hba->irq, hba);
 235                 hba->is_irq_enabled = false;
 236         }
 237 }
 238
 239 /* replace non-printable or non-ASCII characters with spaces */
 240 static inline void ufshcd_remove_non_printable(char *val)
 241 {
 242         if (!val)
 243                 return;
 244
 245         if (*val < 0x20 || *val > 0x7e)
 246                 *val = ' ';
 247 }
 248
 249 /*
 250  * ufshcd_wait_for_register - wait for register value to change
 251  * @hba - per-adapter interface
 252  * @reg - mmio register offset
 253  * @mask - mask to apply to read register value
 254  * @val - wait condition
 255  * @interval_us - polling interval in microsecs
 256  * @timeout_ms - timeout in millisecs
 257  * @can_sleep - perform sleep or just spin
 258  *
 259  * Returns -ETIMEDOUT on error, zero on success
 260  */
 261 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 262                                 u32 val, unsigned long interval_us,
 263                                 unsigned long timeout_ms, bool can_sleep)
 264 {
 265         int err = 0;
 266         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 267
 268         /* ignore bits that we don't intend to wait on */
 269         val = val & mask;
 270
 271         while ((ufshcd_readl(hba, reg) & mask) != val) {
 272                 if (can_sleep)
 273                         usleep_range(interval_us, interval_us + 50);
 274                 else
 275                         udelay(interval_us);
 276                 if (time_after(jiffies, timeout)) {
 277                         if ((ufshcd_readl(hba, reg) & mask) != val)
 278                                 err = -ETIMEDOUT;
 279                         break;
 280                 }
 281         }
 282
 283         return err;
 284 }
 285
 286 /**
 287  * ufshcd_get_intr_mask - Get the interrupt bit mask
 288  * @hba - Pointer to adapter instance
 289  *
 290  * Returns interrupt bit mask per version
 291  */
 292 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 293 {
 294         if (hba->ufs_version == UFSHCI_VERSION_10)
 295                 return INTERRUPT_MASK_ALL_VER_10;
 296         else
 297                 return INTERRUPT_MASK_ALL_VER_11;
 298 }
 299
 300 /**
 301  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 302  * @hba - Pointer to adapter instance
 303  *
 304  * Returns UFSHCI version supported by the controller
 305  */
 306 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 307 {
 308         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 309                 return ufshcd_vops_get_ufs_hci_version(hba);
 310
 311         return ufshcd_readl(hba, REG_UFS_VERSION);
 312 }
 313
 314 /**
 315  * ufshcd_is_device_present - Check if any device connected to
 316  *                            the host controller
 317  * @hba: pointer to adapter instance
 318  *
 319  * Returns 1 if device present, 0 if no device detected
 320  */
 321 static inline int ufshcd_is_device_present(struct ufs_hba *hba)
 322 {
 323         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
 324                                                 DEVICE_PRESENT) ? 1 : 0;
 325 }
 326
 327 /**
 328  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 329  * @lrb: pointer to local command reference block
 330  *
 331  * This function is used to get the OCS field from UTRD
 332  * Returns the OCS field in the UTRD
 333  */
 334 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 335 {
 336         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
 337 }
 338
 339 /**
 340  * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
 341  * @task_req_descp: pointer to utp_task_req_desc structure
 342  *
 343  * This function is used to get the OCS field from UTMRD
 344  * Returns the OCS field in the UTMRD
 345  */
 346 static inline int
 347 ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
 348 {
 349         return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS;
 350 }
 351
 352 /**
 353  * ufshcd_get_tm_free_slot - get a free slot for task management request
 354  * @hba: per adapter instance
 355  * @free_slot: pointer to variable with available slot value
 356  *
 357  * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
 358  * Returns 0 if free slot is not available, else return 1 with tag value
 359  * in @free_slot.
 360  */
 361 static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
 362 {
 363         int tag;
 364         bool ret = false;
 365
 366         if (!free_slot)
 367                 goto out;
 368
 369         do {
 370                 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
 371                 if (tag >= hba->nutmrs)
 372                         goto out;
 373         } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
 374
 375         *free_slot = tag;
 376         ret = true;
 377 out:
 378         return ret;
 379 }
 380
 381 static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
 382 {
 383         clear_bit_unlock(slot, &hba->tm_slots_in_use);
 384 }
 385
 386 /**
 387  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
 388  * @hba: per adapter instance
 389  * @pos: position of the bit to be cleared
 390  */
 391 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 392 {
 393         ufshcd_writel(hba, ~(1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 394 }
 395
 396 /**
 397  * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
 398  * @hba: per adapter instance
 399  * @tag: position of the bit to be cleared
 400  */
 401 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
 402 {
 403         __clear_bit(tag, &hba->outstanding_reqs);
 404 }
 405
 406 /**
 407  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 408  * @reg: Register value of host controller status
 409  *
 410  * Returns integer, 0 on Success and positive value if failed
 411  */
 412 static inline int ufshcd_get_lists_status(u32 reg)
 413 {
 414         /*
 415          * The mask 0xFF is for the following HCS register bits
 416          * Bit          Description
 417          *  0           Device Present
 418          *  1           UTRLRDY
 419          *  2           UTMRLRDY
 420          *  3           UCRDY
 421          * 4-7          reserved
 422          */
 423         return ((reg & 0xFF) >> 1) ^ 0x07;
 424 }
 425
 426 /**
 427  * ufshcd_get_uic_cmd_result - Get the UIC command result
 428  * @hba: Pointer to adapter instance
 429  *
 430  * This function gets the result of UIC command completion
 431  * Returns 0 on success, non zero value on error
 432  */
 433 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 434 {
 435         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 436                MASK_UIC_COMMAND_RESULT;
 437 }
 438
 439 /**
 440  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 441  * @hba: Pointer to adapter instance
 442  *
 443  * This function gets UIC command argument3
 444  * Returns 0 on success, non zero value on error
 445  */
 446 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 447 {
 448         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 449 }
 450
 451 /**
 452  * ufshcd_get_req_rsp - returns the TR response transaction type
 453  * @ucd_rsp_ptr: pointer to response UPIU
 454  */
 455 static inline int
 456 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 457 {
 458         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 459 }
 460
 461 /**
 462  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 463  * @ucd_rsp_ptr: pointer to response UPIU
 464  *
 465  * This function gets the response status and scsi_status from response UPIU
 466  * Returns the response result code.
 467  */
 468 static inline int
 469 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 470 {
 471         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 472 }
 473
 474 /*
 475  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
 476  *                              from response UPIU
 477  * @ucd_rsp_ptr: pointer to response UPIU
 478  *
 479  * Return the data segment length.
 480  */
 481 static inline unsigned int
 482 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
 483 {
 484         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 485                 MASK_RSP_UPIU_DATA_SEG_LEN;
 486 }
 487
 488 /**
 489  * ufshcd_is_exception_event - Check if the device raised an exception event
 490  * @ucd_rsp_ptr: pointer to response UPIU
 491  *
 492  * The function checks if the device raised an exception event indicated in
 493  * the Device Information field of response UPIU.
 494  *
 495  * Returns true if exception is raised, false otherwise.
 496  */
 497 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 498 {
 499         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 500                         MASK_RSP_EXCEPTION_EVENT ? true : false;
 501 }
 502
 503 /**
 504  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 505  * @hba: per adapter instance
 506  */
 507 static inline void
 508 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 509 {
 510         ufshcd_writel(hba, INT_AGGR_ENABLE |
 511                       INT_AGGR_COUNTER_AND_TIMER_RESET,
 512                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 513 }
 514
 515 /**
 516  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 517  * @hba: per adapter instance
 518  * @cnt: Interrupt aggregation counter threshold
 519  * @tmout: Interrupt aggregation timeout value
 520  */
 521 static inline void
 522 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 523 {
 524         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 525                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
 526                       INT_AGGR_TIMEOUT_VAL(tmout),
 527                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 528 }
 529
 530 /**
 531  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 532  * @hba: per adapter instance
 533  */
 534 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 535 {
 536         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 537 }
 538
 539 /**
 540  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 541  *                      When run-stop registers are set to 1, it indicates the
 542  *                      host controller that it can process the requests
 543  * @hba: per adapter instance
 544  */
 545 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 546 {
 547         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 548                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 549         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 550                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 551 }
 552
 553 /**
 554  * ufshcd_hba_start - Start controller initialization sequence
 555  * @hba: per adapter instance
 556  */
 557 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 558 {
 559         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
 560 }
 561
 562 /**
 563  * ufshcd_is_hba_active - Get controller state
 564  * @hba: per adapter instance
 565  *
 566  * Returns zero if controller is active, 1 otherwise
 567  */
 568 static inline int ufshcd_is_hba_active(struct ufs_hba *hba)
 569 {
 570         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1;
 571 }
 572
 573 static void ufshcd_ungate_work(struct work_struct *work)
 574 {
 575         int ret;
 576         unsigned long flags;
 577         struct ufs_hba *hba = container_of(work, struct ufs_hba,
 578                         clk_gating.ungate_work);
 579
 580         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
 581
 582         spin_lock_irqsave(hba->host->host_lock, flags);
 583         if (hba->clk_gating.state == CLKS_ON) {
 584                 spin_unlock_irqrestore(hba->host->host_lock, flags);
 585                 goto unblock_reqs;
 586         }
 587
 588         spin_unlock_irqrestore(hba->host->host_lock, flags);
 589         ufshcd_setup_clocks(hba, true);
 590
 591         /* Exit from hibern8 */
 592         if (ufshcd_can_hibern8_during_gating(hba)) {
 593                 /* Prevent gating in this path */
 594                 hba->clk_gating.is_suspended = true;
 595                 if (ufshcd_is_link_hibern8(hba)) {
 596                         ret = ufshcd_uic_hibern8_exit(hba);
 597                         if (ret)
 598                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
 599                                         __func__, ret);
 600                         else
 601                                 ufshcd_set_link_active(hba);
 602                 }
 603                 hba->clk_gating.is_suspended = false;
 604         }
 605 unblock_reqs:
 606         if (ufshcd_is_clkscaling_enabled(hba))
 607                 devfreq_resume_device(hba->devfreq);
 608         scsi_unblock_requests(hba->host);
 609 }
 610
 611 /**
 612  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
 613  * Also, exit from hibern8 mode and set the link as active.
 614  * @hba: per adapter instance
 615  * @async: This indicates whether caller should ungate clocks asynchronously.
 616  */
 617 int ufshcd_hold(struct ufs_hba *hba, bool async)
 618 {
 619         int rc = 0;
 620         unsigned long flags;
 621
 622         if (!ufshcd_is_clkgating_allowed(hba))
 623                 goto out;
 624         spin_lock_irqsave(hba->host->host_lock, flags);
 625         hba->clk_gating.active_reqs++;
 626
 627         if (ufshcd_eh_in_progress(hba)) {
 628                 spin_unlock_irqrestore(hba->host->host_lock, flags);
 629                 return 0;
 630         }
 631
 632 start:
 633         switch (hba->clk_gating.state) {
 634         case CLKS_ON:
 635                 break;
 636         case REQ_CLKS_OFF:
 637                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
 638                         hba->clk_gating.state = CLKS_ON;
 639                         break;
 640                 }
 641                 /*
 642                  * If we here, it means gating work is either done or
 643                  * currently running. Hence, fall through to cancel gating
 644                  * work and to enable clocks.
 645                  */
 646         case CLKS_OFF:
 647                 scsi_block_requests(hba->host);
 648                 hba->clk_gating.state = REQ_CLKS_ON;
 649                 schedule_work(&hba->clk_gating.ungate_work);
 650                 /*
 651                  * fall through to check if we should wait for this
 652                  * work to be done or not.
 653                  */
 654         case REQ_CLKS_ON:
 655                 if (async) {
 656                         rc = -EAGAIN;
 657                         hba->clk_gating.active_reqs--;
 658                         break;
 659                 }
 660
 661                 spin_unlock_irqrestore(hba->host->host_lock, flags);
 662                 flush_work(&hba->clk_gating.ungate_work);
 663                 /* Make sure state is CLKS_ON before returning */
 664                 spin_lock_irqsave(hba->host->host_lock, flags);
 665                 goto start;
 666         default:
 667                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
 668                                 __func__, hba->clk_gating.state);
 669                 break;
 670         }
 671         spin_unlock_irqrestore(hba->host->host_lock, flags);
 672 out:
 673         return rc;
 674 }
 675 EXPORT_SYMBOL_GPL(ufshcd_hold);
 676
 677 static void ufshcd_gate_work(struct work_struct *work)
 678 {
 679         struct ufs_hba *hba = container_of(work, struct ufs_hba,
 680                         clk_gating.gate_work.work);
 681         unsigned long flags;
 682
 683         spin_lock_irqsave(hba->host->host_lock, flags);
 684         if (hba->clk_gating.is_suspended) {
 685                 hba->clk_gating.state = CLKS_ON;
 686                 goto rel_lock;
 687         }
 688
 689         if (hba->clk_gating.active_reqs
 690                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
 691                 || hba->lrb_in_use || hba->outstanding_tasks
 692                 || hba->active_uic_cmd || hba->uic_async_done)
 693                 goto rel_lock;
 694
 695         spin_unlock_irqrestore(hba->host->host_lock, flags);
 696
 697         /* put the link into hibern8 mode before turning off clocks */
 698         if (ufshcd_can_hibern8_during_gating(hba)) {
 699                 if (ufshcd_uic_hibern8_enter(hba)) {
 700                         hba->clk_gating.state = CLKS_ON;
 701                         goto out;
 702                 }
 703                 ufshcd_set_link_hibern8(hba);
 704         }
 705
 706         if (ufshcd_is_clkscaling_enabled(hba)) {
 707                 devfreq_suspend_device(hba->devfreq);
 708                 hba->clk_scaling.window_start_t = 0;
 709         }
 710
 711         if (!ufshcd_is_link_active(hba))
 712                 ufshcd_setup_clocks(hba, false);
 713         else
 714                 /* If link is active, device ref_clk can't be switched off */
 715                 __ufshcd_setup_clocks(hba, false, true);
 716
 717         /*
 718          * In case you are here to cancel this work the gating state
 719          * would be marked as REQ_CLKS_ON. In this case keep the state
 720          * as REQ_CLKS_ON which would anyway imply that clocks are off
 721          * and a request to turn them on is pending. By doing this way,
 722          * we keep the state machine in tact and this would ultimately
 723          * prevent from doing cancel work multiple times when there are
 724          * new requests arriving before the current cancel work is done.
 725          */
 726         spin_lock_irqsave(hba->host->host_lock, flags);
 727         if (hba->clk_gating.state == REQ_CLKS_OFF)
 728                 hba->clk_gating.state = CLKS_OFF;
 729
 730 rel_lock:
 731         spin_unlock_irqrestore(hba->host->host_lock, flags);
 732 out:
 733         return;
 734 }
 735
 736 /* host lock must be held before calling this variant */
 737 static void __ufshcd_release(struct ufs_hba *hba)
 738 {
 739         if (!ufshcd_is_clkgating_allowed(hba))
 740                 return;
 741
 742         hba->clk_gating.active_reqs--;
 743
 744         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
 745                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
 746                 || hba->lrb_in_use || hba->outstanding_tasks
 747                 || hba->active_uic_cmd || hba->uic_async_done
 748                 || ufshcd_eh_in_progress(hba))
 749                 return;
 750
 751         hba->clk_gating.state = REQ_CLKS_OFF;
 752         schedule_delayed_work(&hba->clk_gating.gate_work,
 753                         msecs_to_jiffies(hba->clk_gating.delay_ms));
 754 }
 755
 756 void ufshcd_release(struct ufs_hba *hba)
 757 {
 758         unsigned long flags;
 759
 760         spin_lock_irqsave(hba->host->host_lock, flags);
 761         __ufshcd_release(hba);
 762         spin_unlock_irqrestore(hba->host->host_lock, flags);
 763 }
 764 EXPORT_SYMBOL_GPL(ufshcd_release);
 765
 766 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
 767                 struct device_attribute *attr, char *buf)
 768 {
 769         struct ufs_hba *hba = dev_get_drvdata(dev);
 770
 771         return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
 772 }
 773
 774 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
 775                 struct device_attribute *attr, const char *buf, size_t count)
 776 {
 777         struct ufs_hba *hba = dev_get_drvdata(dev);
 778         unsigned long flags, value;
 779
 780         if (kstrtoul(buf, 0, &value))
 781                 return -EINVAL;
 782
 783         spin_lock_irqsave(hba->host->host_lock, flags);
 784         hba->clk_gating.delay_ms = value;
 785         spin_unlock_irqrestore(hba->host->host_lock, flags);
 786         return count;
 787 }
 788
 789 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
 790 {
 791         if (!ufshcd_is_clkgating_allowed(hba))
 792                 return;
 793
 794         hba->clk_gating.delay_ms = 150;
 795         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
 796         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
 797
 798         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
 799         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
 800         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
 801         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
 802         hba->clk_gating.delay_attr.attr.mode = S_IRUGO | S_IWUSR;
 803         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
 804                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
 805 }
 806
 807 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
 808 {
 809         if (!ufshcd_is_clkgating_allowed(hba))
 810                 return;
 811         device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
 812         cancel_work_sync(&hba->clk_gating.ungate_work);
 813         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
 814 }
 815
 816 /* Must be called with host lock acquired */
 817 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
 818 {
 819         if (!ufshcd_is_clkscaling_enabled(hba))
 820                 return;
 821
 822         if (!hba->clk_scaling.is_busy_started) {
 823                 hba->clk_scaling.busy_start_t = ktime_get();
 824                 hba->clk_scaling.is_busy_started = true;
 825         }
 826 }
 827
 828 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
 829 {
 830         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
 831
 832         if (!ufshcd_is_clkscaling_enabled(hba))
 833                 return;
 834
 835         if (!hba->outstanding_reqs && scaling->is_busy_started) {
 836                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
 837                                         scaling->busy_start_t));
 838                 scaling->busy_start_t = ktime_set(0, 0);
 839                 scaling->is_busy_started = false;
 840         }
 841 }
 842 /**
 843  * ufshcd_send_command - Send SCSI or device management commands
 844  * @hba: per adapter instance
 845  * @task_tag: Task tag of the command
 846  */
 847 static inline
 848 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
 849 {
 850         ufshcd_clk_scaling_start_busy(hba);
 851         __set_bit(task_tag, &hba->outstanding_reqs);
 852         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 853 }
 854
 855 /**
 856  * ufshcd_copy_sense_data - Copy sense data in case of check condition
 857  * @lrb - pointer to local reference block
 858  */
 859 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
 860 {
 861         int len;
 862         if (lrbp->sense_buffer &&
 863             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
 864                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
 865                 memcpy(lrbp->sense_buffer,
 866                         lrbp->ucd_rsp_ptr->sr.sense_data,
 867                         min_t(int, len, SCSI_SENSE_BUFFERSIZE));
 868         }
 869 }
 870
 871 /**
 872  * ufshcd_copy_query_response() - Copy the Query Response and the data
 873  * descriptor
 874  * @hba: per adapter instance
 875  * @lrb - pointer to local reference block
 876  */
 877 static
 878 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
 879 {
 880         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
 881
 882         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
 883
 884         /* Get the descriptor */
 885         if (lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
 886                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
 887                                 GENERAL_UPIU_REQUEST_SIZE;
 888                 u16 resp_len;
 889                 u16 buf_len;
 890
 891                 /* data segment length */
 892                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
 893                                                 MASK_QUERY_DATA_SEG_LEN;
 894                 buf_len = be16_to_cpu(
 895                                 hba->dev_cmd.query.request.upiu_req.length);
 896                 if (likely(buf_len >= resp_len)) {
 897                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
 898                 } else {
 899                         dev_warn(hba->dev,
 900                                 "%s: Response size is bigger than buffer",
 901                                 __func__);
 902                         return -EINVAL;
 903                 }
 904         }
 905
 906         return 0;
 907 }
 908
 909 /**
 910  * ufshcd_hba_capabilities - Read controller capabilities
 911  * @hba: per adapter instance
 912  */
 913 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
 914 {
 915         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
 916
 917         /* nutrs and nutmrs are 0 based values */
 918         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
 919         hba->nutmrs =
 920         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
 921 }
 922
 923 /**
 924  * ufshcd_ready_for_uic_cmd - Check if controller is ready
 925  *                            to accept UIC commands
 926  * @hba: per adapter instance
 927  * Return true on success, else false
 928  */
 929 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
 930 {
 931         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
 932                 return true;
 933         else
 934                 return false;
 935 }
 936
 937 /**
 938  * ufshcd_get_upmcrs - Get the power mode change request status
 939  * @hba: Pointer to adapter instance
 940  *
 941  * This function gets the UPMCRS field of HCS register
 942  * Returns value of UPMCRS field
 943  */
 944 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
 945 {
 946         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
 947 }
 948
 949 /**
 950  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
 951  * @hba: per adapter instance
 952  * @uic_cmd: UIC command
 953  *
 954  * Mutex must be held.
 955  */
 956 static inline void
 957 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 958 {
 959         WARN_ON(hba->active_uic_cmd);
 960
 961         hba->active_uic_cmd = uic_cmd;
 962
 963         /* Write Args */
 964         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
 965         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
 966         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
 967
 968         /* Write UIC Cmd */
 969         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
 970                       REG_UIC_COMMAND);
 971 }
 972
 973 /**
 974  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
 975  * @hba: per adapter instance
 976  * @uic_command: UIC command
 977  *
 978  * Must be called with mutex held.
 979  * Returns 0 only if success.
 980  */
 981 static int
 982 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 983 {
 984         int ret;
 985         unsigned long flags;
 986
 987         if (wait_for_completion_timeout(&uic_cmd->done,
 988                                         msecs_to_jiffies(UIC_CMD_TIMEOUT)))
 989                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
 990         else
 991                 ret = -ETIMEDOUT;
 992
 993         spin_lock_irqsave(hba->host->host_lock, flags);
 994         hba->active_uic_cmd = NULL;
 995         spin_unlock_irqrestore(hba->host->host_lock, flags);
 996
 997         return ret;
 998 }
 999
1000 /**
1001  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1002  * @hba: per adapter instance
1003  * @uic_cmd: UIC command
1004  * @completion: initialize the completion only if this is set to true
1005  *
1006  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
1007  * with mutex held and host_lock locked.
1008  * Returns 0 only if success.
1009  */
1010 static int
1011 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
1012                       bool completion)
1013 {
1014         if (!ufshcd_ready_for_uic_cmd(hba)) {
1015                 dev_err(hba->dev,
1016                         "Controller not ready to accept UIC commands\n");
1017                 return -EIO;
1018         }
1019
1020         if (completion)
1021                 init_completion(&uic_cmd->done);
1022
1023         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
1024
1025         return 0;
1026 }
1027
1028 /**
1029  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1030  * @hba: per adapter instance
1031  * @uic_cmd: UIC command
1032  *
1033  * Returns 0 only if success.
1034  */
1035 static int
1036 ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
1037 {
1038         int ret;
1039         unsigned long flags;
1040
1041         ufshcd_hold(hba, false);
1042         mutex_lock(&hba->uic_cmd_mutex);
1043         ufshcd_add_delay_before_dme_cmd(hba);
1044
1045         spin_lock_irqsave(hba->host->host_lock, flags);
1046         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
1047         spin_unlock_irqrestore(hba->host->host_lock, flags);
1048         if (!ret)
1049                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
1050
1051         mutex_unlock(&hba->uic_cmd_mutex);
1052
1053         ufshcd_release(hba);
1054         return ret;
1055 }
1056
1057 /**
1058  * ufshcd_map_sg - Map scatter-gather list to prdt
1059  * @lrbp - pointer to local reference block
1060  *
1061  * Returns 0 in case of success, non-zero value in case of failure
1062  */
1063 static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
1064 {
1065         struct ufshcd_sg_entry *prd_table;
1066         struct scatterlist *sg;
1067         struct scsi_cmnd *cmd;
1068         int sg_segments;
1069         int i;
1070
1071         cmd = lrbp->cmd;
1072         sg_segments = scsi_dma_map(cmd);
1073         if (sg_segments < 0)
1074                 return sg_segments;
1075
1076         if (sg_segments) {
1077                 lrbp->utr_descriptor_ptr->prd_table_length =
1078                                         cpu_to_le16((u16) (sg_segments));
1079
1080                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
1081
1082                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
1083                         prd_table[i].size  =
1084                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
1085                         prd_table[i].base_addr =
1086                                 cpu_to_le32(lower_32_bits(sg->dma_address));
1087                         prd_table[i].upper_addr =
1088                                 cpu_to_le32(upper_32_bits(sg->dma_address));
1089                         prd_table[i].reserved = 0;
1090                 }
1091         } else {
1092                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
1093         }
1094
1095         return 0;
1096 }
1097
1098 /**
1099  * ufshcd_enable_intr - enable interrupts
1100  * @hba: per adapter instance
1101  * @intrs: interrupt bits
1102  */
1103 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
1104 {
1105         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
1106
1107         if (hba->ufs_version == UFSHCI_VERSION_10) {
1108                 u32 rw;
1109                 rw = set & INTERRUPT_MASK_RW_VER_10;
1110                 set = rw | ((set ^ intrs) & intrs);
1111         } else {
1112                 set |= intrs;
1113         }
1114
1115         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
1116 }
1117
1118 /**
1119  * ufshcd_disable_intr - disable interrupts
1120  * @hba: per adapter instance
1121  * @intrs: interrupt bits
1122  */
1123 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
1124 {
1125         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
1126
1127         if (hba->ufs_version == UFSHCI_VERSION_10) {
1128                 u32 rw;
1129                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
1130                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
1131                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
1132
1133         } else {
1134                 set &= ~intrs;
1135         }
1136
1137         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
1138 }
1139
1140 /**
1141  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
1142  * descriptor according to request
1143  * @lrbp: pointer to local reference block
1144  * @upiu_flags: flags required in the header
1145  * @cmd_dir: requests data direction
1146  */
1147 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
1148                 u32 *upiu_flags, enum dma_data_direction cmd_dir)
1149 {
1150         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
1151         u32 data_direction;
1152         u32 dword_0;
1153
1154         if (cmd_dir == DMA_FROM_DEVICE) {
1155                 data_direction = UTP_DEVICE_TO_HOST;
1156                 *upiu_flags = UPIU_CMD_FLAGS_READ;
1157         } else if (cmd_dir == DMA_TO_DEVICE) {
1158                 data_direction = UTP_HOST_TO_DEVICE;
1159                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
1160         } else {
1161                 data_direction = UTP_NO_DATA_TRANSFER;
1162                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
1163         }
1164
1165         dword_0 = data_direction | (lrbp->command_type
1166                                 << UPIU_COMMAND_TYPE_OFFSET);
1167         if (lrbp->intr_cmd)
1168                 dword_0 |= UTP_REQ_DESC_INT_CMD;
1169
1170         /* Transfer request descriptor header fields */
1171         req_desc->header.dword_0 = cpu_to_le32(dword_0);
1172         /* dword_1 is reserved, hence it is set to 0 */
1173         req_desc->header.dword_1 = 0;
1174         /*
1175          * assigning invalid value for command status. Controller
1176          * updates OCS on command completion, with the command
1177          * status
1178          */
1179         req_desc->header.dword_2 =
1180                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
1181         /* dword_3 is reserved, hence it is set to 0 */
1182         req_desc->header.dword_3 = 0;
1183
1184         req_desc->prd_table_length = 0;
1185 }
1186
1187 /**
1188  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
1189  * for scsi commands
1190  * @lrbp - local reference block pointer
1191  * @upiu_flags - flags
1192  */
1193 static
1194 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
1195 {
1196         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1197         unsigned short cdb_len;
1198
1199         /* command descriptor fields */
1200         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
1201                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
1202                                 lrbp->lun, lrbp->task_tag);
1203         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
1204                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
1205
1206         /* Total EHS length and Data segment length will be zero */
1207         ucd_req_ptr->header.dword_2 = 0;
1208
1209         ucd_req_ptr->sc.exp_data_transfer_len =
1210                 cpu_to_be32(lrbp->cmd->sdb.length);
1211
1212         cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE);
1213         memset(ucd_req_ptr->sc.cdb, 0, MAX_CDB_SIZE);
1214         memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
1215
1216         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1217 }
1218
1219 /**
1220  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
1221  * for query requsts
1222  * @hba: UFS hba
1223  * @lrbp: local reference block pointer
1224  * @upiu_flags: flags
1225  */
1226 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
1227                                 struct ufshcd_lrb *lrbp, u32 upiu_flags)
1228 {
1229         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1230         struct ufs_query *query = &hba->dev_cmd.query;
1231         u16 len = be16_to_cpu(query->request.upiu_req.length);
1232         u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
1233
1234         /* Query request header */
1235         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
1236                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
1237                         lrbp->lun, lrbp->task_tag);
1238         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
1239                         0, query->request.query_func, 0, 0);
1240
1241         /* Data segment length */
1242         ucd_req_ptr->header.dword_2 = UPIU_HEADER_DWORD(
1243                         0, 0, len >> 8, (u8)len);
1244
1245         /* Copy the Query Request buffer as is */
1246         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
1247                         QUERY_OSF_SIZE);
1248
1249         /* Copy the Descriptor */
1250         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
1251                 memcpy(descp, query->descriptor, len);
1252
1253         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1254 }
1255
1256 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
1257 {
1258         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
1259
1260         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
1261
1262         /* command descriptor fields */
1263         ucd_req_ptr->header.dword_0 =
1264                 UPIU_HEADER_DWORD(
1265                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
1266         /* clear rest of the fields of basic header */
1267         ucd_req_ptr->header.dword_1 = 0;
1268         ucd_req_ptr->header.dword_2 = 0;
1269
1270         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1271 }
1272
1273 /**
1274  * ufshcd_compose_upiu - form UFS Protocol Information Unit(UPIU)
1275  * @hba - per adapter instance
1276  * @lrb - pointer to local reference block
1277  */
1278 static int ufshcd_compose_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1279 {
1280         u32 upiu_flags;
1281         int ret = 0;
1282
1283         switch (lrbp->command_type) {
1284         case UTP_CMD_TYPE_SCSI:
1285                 if (likely(lrbp->cmd)) {
1286                         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
1287                                         lrbp->cmd->sc_data_direction);
1288                         ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
1289                 } else {
1290                         ret = -EINVAL;
1291                 }
1292                 break;
1293         case UTP_CMD_TYPE_DEV_MANAGE:
1294                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
1295                 if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
1296                         ufshcd_prepare_utp_query_req_upiu(
1297                                         hba, lrbp, upiu_flags);
1298                 else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
1299                         ufshcd_prepare_utp_nop_upiu(lrbp);
1300                 else
1301                         ret = -EINVAL;
1302                 break;
1303         case UTP_CMD_TYPE_UFS:
1304                 /* For UFS native command implementation */
1305                 ret = -ENOTSUPP;
1306                 dev_err(hba->dev, "%s: UFS native command are not supported\n",
1307                         __func__);
1308                 break;
1309         default:
1310                 ret = -ENOTSUPP;
1311                 dev_err(hba->dev, "%s: unknown command type: 0x%x\n",
1312                                 __func__, lrbp->command_type);
1313                 break;
1314         } /* end of switch */
1315
1316         return ret;
1317 }
1318
1319 /*
1320  * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN
1321  * @scsi_lun: scsi LUN id
1322  *
1323  * Returns UPIU LUN id
1324  */
1325 static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun)
1326 {
1327         if (scsi_is_wlun(scsi_lun))
1328                 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID)
1329                         | UFS_UPIU_WLUN_ID;
1330         else
1331                 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID;
1332 }
1333
1334 /**
1335  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
1336  * @scsi_lun: UPIU W-LUN id
1337  *
1338  * Returns SCSI W-LUN id
1339  */
1340 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
1341 {
1342         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
1343 }
1344
1345 /**
1346  * ufshcd_queuecommand - main entry point for SCSI requests
1347  * @cmd: command from SCSI Midlayer
1348  * @done: call back function
1349  *
1350  * Returns 0 for success, non-zero in case of failure
1351  */
1352 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
1353 {
1354         struct ufshcd_lrb *lrbp;
1355         struct ufs_hba *hba;
1356         unsigned long flags;
1357         int tag;
1358         int err = 0;
1359
1360         hba = shost_priv(host);
1361
1362         tag = cmd->request->tag;
1363         if (!ufshcd_valid_tag(hba, tag)) {
1364                 dev_err(hba->dev,
1365                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
1366                         __func__, tag, cmd, cmd->request);
1367                 BUG();
1368         }
1369
1370         spin_lock_irqsave(hba->host->host_lock, flags);
1371         switch (hba->ufshcd_state) {
1372         case UFSHCD_STATE_OPERATIONAL:
1373                 break;
1374         case UFSHCD_STATE_RESET:
1375                 err = SCSI_MLQUEUE_HOST_BUSY;
1376                 goto out_unlock;
1377         case UFSHCD_STATE_ERROR:
1378                 set_host_byte(cmd, DID_ERROR);
1379                 cmd->scsi_done(cmd);
1380                 goto out_unlock;
1381         default:
1382                 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
1383                                 __func__, hba->ufshcd_state);
1384                 set_host_byte(cmd, DID_BAD_TARGET);
1385                 cmd->scsi_done(cmd);
1386                 goto out_unlock;
1387         }
1388
1389         /* if error handling is in progress, don't issue commands */
1390         if (ufshcd_eh_in_progress(hba)) {
1391                 set_host_byte(cmd, DID_ERROR);
1392                 cmd->scsi_done(cmd);
1393                 goto out_unlock;
1394         }
1395         spin_unlock_irqrestore(hba->host->host_lock, flags);
1396
1397         /* acquire the tag to make sure device cmds don't use it */
1398         if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
1399                 /*
1400                  * Dev manage command in progress, requeue the command.
1401                  * Requeuing the command helps in cases where the request *may*
1402                  * find different tag instead of waiting for dev manage command
1403                  * completion.
1404                  */
1405                 err = SCSI_MLQUEUE_HOST_BUSY;
1406                 goto out;
1407         }
1408
1409         err = ufshcd_hold(hba, true);
1410         if (err) {
1411                 err = SCSI_MLQUEUE_HOST_BUSY;
1412                 clear_bit_unlock(tag, &hba->lrb_in_use);
1413                 goto out;
1414         }
1415         WARN_ON(hba->clk_gating.state != CLKS_ON);
1416
1417         lrbp = &hba->lrb[tag];
1418
1419         WARN_ON(lrbp->cmd);
1420         lrbp->cmd = cmd;
1421         lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
1422         lrbp->sense_buffer = cmd->sense_buffer;
1423         lrbp->task_tag = tag;
1424         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
1425         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
1426         lrbp->command_type = UTP_CMD_TYPE_SCSI;
1427
1428         /* form UPIU before issuing the command */
1429         ufshcd_compose_upiu(hba, lrbp);
1430         err = ufshcd_map_sg(lrbp);
1431         if (err) {
1432                 lrbp->cmd = NULL;
1433                 clear_bit_unlock(tag, &hba->lrb_in_use);
1434                 goto out;
1435         }
1436
1437         /* issue command to the controller */
1438         spin_lock_irqsave(hba->host->host_lock, flags);
1439         ufshcd_send_command(hba, tag);
1440 out_unlock:
1441         spin_unlock_irqrestore(hba->host->host_lock, flags);
1442 out:
1443         return err;
1444 }
1445
1446 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
1447                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
1448 {
1449         lrbp->cmd = NULL;
1450         lrbp->sense_bufflen = 0;
1451         lrbp->sense_buffer = NULL;
1452         lrbp->task_tag = tag;
1453         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
1454         lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
1455         lrbp->intr_cmd = true; /* No interrupt aggregation */
1456         hba->dev_cmd.type = cmd_type;
1457
1458         return ufshcd_compose_upiu(hba, lrbp);
1459 }
1460
1461 static int
1462 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
1463 {
1464         int err = 0;
1465         unsigned long flags;
1466         u32 mask = 1 << tag;
1467
1468         /* clear outstanding transaction before retry */
1469         spin_lock_irqsave(hba->host->host_lock, flags);
1470         ufshcd_utrl_clear(hba, tag);
1471         spin_unlock_irqrestore(hba->host->host_lock, flags);
1472
1473         /*
1474          * wait for for h/w to clear corresponding bit in door-bell.
1475          * max. wait is 1 sec.
1476          */
1477         err = ufshcd_wait_for_register(hba,
1478                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
1479                         mask, ~mask, 1000, 1000, true);
1480
1481         return err;
1482 }
1483
1484 static int
1485 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1486 {
1487         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1488
1489         /* Get the UPIU response */
1490         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
1491                                 UPIU_RSP_CODE_OFFSET;
1492         return query_res->response;
1493 }
1494
1495 /**
1496  * ufshcd_dev_cmd_completion() - handles device management command responses
1497  * @hba: per adapter instance
1498  * @lrbp: pointer to local reference block
1499  */
1500 static int
1501 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1502 {
1503         int resp;
1504         int err = 0;
1505
1506         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
1507
1508         switch (resp) {
1509         case UPIU_TRANSACTION_NOP_IN:
1510                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
1511                         err = -EINVAL;
1512                         dev_err(hba->dev, "%s: unexpected response %x\n",
1513                                         __func__, resp);
1514                 }
1515                 break;
1516         case UPIU_TRANSACTION_QUERY_RSP:
1517                 err = ufshcd_check_query_response(hba, lrbp);
1518                 if (!err)
1519                         err = ufshcd_copy_query_response(hba, lrbp);
1520                 break;
1521         case UPIU_TRANSACTION_REJECT_UPIU:
1522                 /* TODO: handle Reject UPIU Response */
1523                 err = -EPERM;
1524                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
1525                                 __func__);
1526                 break;
1527         default:
1528                 err = -EINVAL;
1529                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
1530                                 __func__, resp);
1531                 break;
1532         }
1533
1534         return err;
1535 }
1536
1537 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
1538                 struct ufshcd_lrb *lrbp, int max_timeout)
1539 {
1540         int err = 0;
1541         unsigned long time_left;
1542         unsigned long flags;
1543
1544         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
1545                         msecs_to_jiffies(max_timeout));
1546
1547         spin_lock_irqsave(hba->host->host_lock, flags);
1548         hba->dev_cmd.complete = NULL;
1549         if (likely(time_left)) {
1550                 err = ufshcd_get_tr_ocs(lrbp);
1551                 if (!err)
1552                         err = ufshcd_dev_cmd_completion(hba, lrbp);
1553         }
1554         spin_unlock_irqrestore(hba->host->host_lock, flags);
1555
1556         if (!time_left) {
1557                 err = -ETIMEDOUT;
1558                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
1559                         __func__, lrbp->task_tag);
1560                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
1561                         /* successfully cleared the command, retry if needed */
1562                         err = -EAGAIN;
1563                 /*
1564                  * in case of an error, after clearing the doorbell,
1565                  * we also need to clear the outstanding_request
1566                  * field in hba
1567                  */
1568                 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
1569         }
1570
1571         return err;
1572 }
1573
1574 /**
1575  * ufshcd_get_dev_cmd_tag - Get device management command tag
1576  * @hba: per-adapter instance
1577  * @tag: pointer to variable with available slot value
1578  *
1579  * Get a free slot and lock it until device management command
1580  * completes.
1581  *
1582  * Returns false if free slot is unavailable for locking, else
1583  * return true with tag value in @tag.
1584  */
1585 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
1586 {
1587         int tag;
1588         bool ret = false;
1589         unsigned long tmp;
1590
1591         if (!tag_out)
1592                 goto out;
1593
1594         do {
1595                 tmp = ~hba->lrb_in_use;
1596                 tag = find_last_bit(&tmp, hba->nutrs);
1597                 if (tag >= hba->nutrs)
1598                         goto out;
1599         } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
1600
1601         *tag_out = tag;
1602         ret = true;
1603 out:
1604         return ret;
1605 }
1606
1607 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
1608 {
1609         clear_bit_unlock(tag, &hba->lrb_in_use);
1610 }
1611
1612 /**
1613  * ufshcd_exec_dev_cmd - API for sending device management requests
1614  * @hba - UFS hba
1615  * @cmd_type - specifies the type (NOP, Query...)
1616  * @timeout - time in seconds
1617  *
1618  * NOTE: Since there is only one available tag for device management commands,
1619  * it is expected you hold the hba->dev_cmd.lock mutex.
1620  */
1621 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
1622                 enum dev_cmd_type cmd_type, int timeout)
1623 {
1624         struct ufshcd_lrb *lrbp;
1625         int err;
1626         int tag;
1627         struct completion wait;
1628         unsigned long flags;
1629
1630         /*
1631          * Get free slot, sleep if slots are unavailable.
1632          * Even though we use wait_event() which sleeps indefinitely,
1633          * the maximum wait time is bounded by SCSI request timeout.
1634          */
1635         wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
1636
1637         init_completion(&wait);
1638         lrbp = &hba->lrb[tag];
1639         WARN_ON(lrbp->cmd);
1640         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
1641         if (unlikely(err))
1642                 goto out_put_tag;
1643
1644         hba->dev_cmd.complete = &wait;
1645
1646         /* Make sure descriptors are ready before ringing the doorbell */
1647         wmb();
1648         spin_lock_irqsave(hba->host->host_lock, flags);
1649         ufshcd_send_command(hba, tag);
1650         spin_unlock_irqrestore(hba->host->host_lock, flags);
1651
1652         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
1653
1654 out_put_tag:
1655         ufshcd_put_dev_cmd_tag(hba, tag);
1656         wake_up(&hba->dev_cmd.tag_wq);
1657         return err;
1658 }
1659
1660 /**
1661  * ufshcd_init_query() - init the query response and request parameters
1662  * @hba: per-adapter instance
1663  * @request: address of the request pointer to be initialized
1664  * @response: address of the response pointer to be initialized
1665  * @opcode: operation to perform
1666  * @idn: flag idn to access
1667  * @index: LU number to access
1668  * @selector: query/flag/descriptor further identification
1669  */
1670 static inline void ufshcd_init_query(struct ufs_hba *hba,
1671                 struct ufs_query_req **request, struct ufs_query_res **response,
1672                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
1673 {
1674         *request = &hba->dev_cmd.query.request;
1675         *response = &hba->dev_cmd.query.response;
1676         memset(*request, 0, sizeof(struct ufs_query_req));
1677         memset(*response, 0, sizeof(struct ufs_query_res));
1678         (*request)->upiu_req.opcode = opcode;
1679         (*request)->upiu_req.idn = idn;
1680         (*request)->upiu_req.index = index;
1681         (*request)->upiu_req.selector = selector;
1682 }
1683
1684 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1685         enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
1686 {
1687         int ret;
1688         int retries;
1689
1690         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1691                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
1692                 if (ret)
1693                         dev_dbg(hba->dev,
1694                                 "%s: failed with error %d, retries %d\n",
1695                                 __func__, ret, retries);
1696                 else
1697                         break;
1698         }
1699
1700         if (ret)
1701                 dev_err(hba->dev,
1702                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
1703                         __func__, opcode, idn, ret, retries);
1704         return ret;
1705 }
1706
1707 /**
1708  * ufshcd_query_flag() - API function for sending flag query requests
1709  * hba: per-adapter instance
1710  * query_opcode: flag query to perform
1711  * idn: flag idn to access
1712  * flag_res: the flag value after the query request completes
1713  *
1714  * Returns 0 for success, non-zero in case of failure
1715  */
1716 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1717                         enum flag_idn idn, bool *flag_res)
1718 {
1719         struct ufs_query_req *request = NULL;
1720         struct ufs_query_res *response = NULL;
1721         int err, index = 0, selector = 0;
1722         int timeout = QUERY_REQ_TIMEOUT;
1723
1724         BUG_ON(!hba);
1725
1726         ufshcd_hold(hba, false);
1727         mutex_lock(&hba->dev_cmd.lock);
1728         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1729                         selector);
1730
1731         switch (opcode) {
1732         case UPIU_QUERY_OPCODE_SET_FLAG:
1733         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
1734         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
1735                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1736                 break;
1737         case UPIU_QUERY_OPCODE_READ_FLAG:
1738                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1739                 if (!flag_res) {
1740                         /* No dummy reads */
1741                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
1742                                         __func__);
1743                         err = -EINVAL;
1744                         goto out_unlock;
1745                 }
1746                 break;
1747         default:
1748                 dev_err(hba->dev,
1749                         "%s: Expected query flag opcode but got = %d\n",
1750                         __func__, opcode);
1751                 err = -EINVAL;
1752                 goto out_unlock;
1753         }
1754
1755         if (idn == QUERY_FLAG_IDN_FDEVICEINIT)
1756                 timeout = QUERY_FDEVICEINIT_REQ_TIMEOUT;
1757
1758         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
1759
1760         if (err) {
1761                 dev_err(hba->dev,
1762                         "%s: Sending flag query for idn %d failed, err = %d\n",
1763                         __func__, idn, err);
1764                 goto out_unlock;
1765         }
1766
1767         if (flag_res)
1768                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
1769                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
1770
1771 out_unlock:
1772         mutex_unlock(&hba->dev_cmd.lock);
1773         ufshcd_release(hba);
1774         return err;
1775 }
1776
1777 /**
1778  * ufshcd_query_attr - API function for sending attribute requests
1779  * hba: per-adapter instance
1780  * opcode: attribute opcode
1781  * idn: attribute idn to access
1782  * index: index field
1783  * selector: selector field
1784  * attr_val: the attribute value after the query request completes
1785  *
1786  * Returns 0 for success, non-zero in case of failure
1787 */
1788 static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
1789                         enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
1790 {
1791         struct ufs_query_req *request = NULL;
1792         struct ufs_query_res *response = NULL;
1793         int err;
1794
1795         BUG_ON(!hba);
1796
1797         ufshcd_hold(hba, false);
1798         if (!attr_val) {
1799                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
1800                                 __func__, opcode);
1801                 err = -EINVAL;
1802                 goto out;
1803         }
1804
1805         mutex_lock(&hba->dev_cmd.lock);
1806         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1807                         selector);
1808
1809         switch (opcode) {
1810         case UPIU_QUERY_OPCODE_WRITE_ATTR:
1811                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1812                 request->upiu_req.value = cpu_to_be32(*attr_val);
1813                 break;
1814         case UPIU_QUERY_OPCODE_READ_ATTR:
1815                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1816                 break;
1817         default:
1818                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
1819                                 __func__, opcode);
1820                 err = -EINVAL;
1821                 goto out_unlock;
1822         }
1823
1824         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1825
1826         if (err) {
1827                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1828                                 __func__, opcode, idn, err);
1829                 goto out_unlock;
1830         }
1831
1832         *attr_val = be32_to_cpu(response->upiu_res.value);
1833
1834 out_unlock:
1835         mutex_unlock(&hba->dev_cmd.lock);
1836 out:
1837         ufshcd_release(hba);
1838         return err;
1839 }
1840
1841 /**
1842  * ufshcd_query_attr_retry() - API function for sending query
1843  * attribute with retries
1844  * @hba: per-adapter instance
1845  * @opcode: attribute opcode
1846  * @idn: attribute idn to access
1847  * @index: index field
1848  * @selector: selector field
1849  * @attr_val: the attribute value after the query request
1850  * completes
1851  *
1852  * Returns 0 for success, non-zero in case of failure
1853 */
1854 static int ufshcd_query_attr_retry(struct ufs_hba *hba,
1855         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
1856         u32 *attr_val)
1857 {
1858         int ret = 0;
1859         u32 retries;
1860
1861          for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1862                 ret = ufshcd_query_attr(hba, opcode, idn, index,
1863                                                 selector, attr_val);
1864                 if (ret)
1865                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
1866                                 __func__, ret, retries);
1867                 else
1868                         break;
1869         }
1870
1871         if (ret)
1872                 dev_err(hba->dev,
1873                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
1874                         __func__, idn, ret, QUERY_REQ_RETRIES);
1875         return ret;
1876 }
1877
1878 /**
1879  * ufshcd_query_descriptor - API function for sending descriptor requests
1880  * hba: per-adapter instance
1881  * opcode: attribute opcode
1882  * idn: attribute idn to access
1883  * index: index field
1884  * selector: selector field
1885  * desc_buf: the buffer that contains the descriptor
1886  * buf_len: length parameter passed to the device
1887  *
1888  * Returns 0 for success, non-zero in case of failure.
1889  * The buf_len parameter will contain, on return, the length parameter
1890  * received on the response.
1891  */
1892 static int ufshcd_query_descriptor(struct ufs_hba *hba,
1893                         enum query_opcode opcode, enum desc_idn idn, u8 index,
1894                         u8 selector, u8 *desc_buf, int *buf_len)
1895 {
1896         struct ufs_query_req *request = NULL;
1897         struct ufs_query_res *response = NULL;
1898         int err;
1899
1900         BUG_ON(!hba);
1901
1902         ufshcd_hold(hba, false);
1903         if (!desc_buf) {
1904                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
1905                                 __func__, opcode);
1906                 err = -EINVAL;
1907                 goto out;
1908         }
1909
1910         if (*buf_len <= QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
1911                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
1912                                 __func__, *buf_len);
1913                 err = -EINVAL;
1914                 goto out;
1915         }
1916
1917         mutex_lock(&hba->dev_cmd.lock);
1918         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1919                         selector);
1920         hba->dev_cmd.query.descriptor = desc_buf;
1921         request->upiu_req.length = cpu_to_be16(*buf_len);
1922
1923         switch (opcode) {
1924         case UPIU_QUERY_OPCODE_WRITE_DESC:
1925                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1926                 break;
1927         case UPIU_QUERY_OPCODE_READ_DESC:
1928                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1929                 break;
1930         default:
1931                 dev_err(hba->dev,
1932                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
1933                                 __func__, opcode);
1934                 err = -EINVAL;
1935                 goto out_unlock;
1936         }
1937
1938         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1939
1940         if (err) {
1941                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
1942                                 __func__, opcode, idn, err);
1943                 goto out_unlock;
1944         }
1945
1946         hba->dev_cmd.query.descriptor = NULL;
1947         *buf_len = be16_to_cpu(response->upiu_res.length);
1948
1949 out_unlock:
1950         mutex_unlock(&hba->dev_cmd.lock);
1951 out:
1952         ufshcd_release(hba);
1953         return err;
1954 }
1955
1956 /**
1957  * ufshcd_read_desc_param - read the specified descriptor parameter
1958  * @hba: Pointer to adapter instance
1959  * @desc_id: descriptor idn value
1960  * @desc_index: descriptor index
1961  * @param_offset: offset of the parameter to read
1962  * @param_read_buf: pointer to buffer where parameter would be read
1963  * @param_size: sizeof(param_read_buf)
1964  *
1965  * Return 0 in case of success, non-zero otherwise
1966  */
1967 static int ufshcd_read_desc_param(struct ufs_hba *hba,
1968                                   enum desc_idn desc_id,
1969                                   int desc_index,
1970                                   u32 param_offset,
1971                                   u8 *param_read_buf,
1972                                   u32 param_size)
1973 {
1974         int ret;
1975         u8 *desc_buf;
1976         u32 buff_len;
1977         bool is_kmalloc = true;
1978
1979         /* safety checks */
1980         if (desc_id >= QUERY_DESC_IDN_MAX)
1981                 return -EINVAL;
1982
1983         buff_len = ufs_query_desc_max_size[desc_id];
1984         if ((param_offset + param_size) > buff_len)
1985                 return -EINVAL;
1986
1987         if (!param_offset && (param_size == buff_len)) {
1988                 /* memory space already available to hold full descriptor */
1989                 desc_buf = param_read_buf;
1990                 is_kmalloc = false;
1991         } else {
1992                 /* allocate memory to hold full descriptor */
1993                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
1994                 if (!desc_buf)
1995                         return -ENOMEM;
1996         }
1997
1998         ret = ufshcd_query_descriptor(hba, UPIU_QUERY_OPCODE_READ_DESC,
1999                                       desc_id, desc_index, 0, desc_buf,
2000                                       &buff_len);
2001
2002         if (ret || (buff_len < ufs_query_desc_max_size[desc_id]) ||
2003             (desc_buf[QUERY_DESC_LENGTH_OFFSET] !=
2004              ufs_query_desc_max_size[desc_id])
2005             || (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id)) {
2006                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d param_offset %d buff_len %d ret %d",
2007                         __func__, desc_id, param_offset, buff_len, ret);
2008                 if (!ret)
2009                         ret = -EINVAL;
2010
2011                 goto out;
2012         }
2013
2014         if (is_kmalloc)
2015                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
2016 out:
2017         if (is_kmalloc)
2018                 kfree(desc_buf);
2019         return ret;
2020 }
2021
2022 static inline int ufshcd_read_desc(struct ufs_hba *hba,
2023                                    enum desc_idn desc_id,
2024                                    int desc_index,
2025                                    u8 *buf,
2026                                    u32 size)
2027 {
2028         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
2029 }
2030
2031 static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
2032                                          u8 *buf,
2033                                          u32 size)
2034 {
2035         return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
2036 }
2037
2038 int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
2039 {
2040         return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
2041 }
2042 EXPORT_SYMBOL(ufshcd_read_device_desc);
2043
2044 /**
2045  * ufshcd_read_string_desc - read string descriptor
2046  * @hba: pointer to adapter instance
2047  * @desc_index: descriptor index
2048  * @buf: pointer to buffer where descriptor would be read
2049  * @size: size of buf
2050  * @ascii: if true convert from unicode to ascii characters
2051  *
2052  * Return 0 in case of success, non-zero otherwise
2053  */
2054 int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index, u8 *buf,
2055                                 u32 size, bool ascii)
2056 {
2057         int err = 0;
2058
2059         err = ufshcd_read_desc(hba,
2060                                 QUERY_DESC_IDN_STRING, desc_index, buf, size);
2061
2062         if (err) {
2063                 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
2064                         __func__, QUERY_REQ_RETRIES, err);
2065                 goto out;
2066         }
2067
2068         if (ascii) {
2069                 int desc_len;
2070                 int ascii_len;
2071                 int i;
2072                 char *buff_ascii;
2073
2074                 desc_len = buf[0];
2075                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
2076                 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
2077                 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
2078                         dev_err(hba->dev, "%s: buffer allocated size is too small\n",
2079                                         __func__);
2080                         err = -ENOMEM;
2081                         goto out;
2082                 }
2083
2084                 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
2085                 if (!buff_ascii) {
2086                         err = -ENOMEM;
2087                         goto out_free_buff;
2088                 }
2089
2090                 /*
2091                  * the descriptor contains string in UTF16 format
2092                  * we need to convert to utf-8 so it can be displayed
2093                  */
2094                 utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
2095                                 desc_len - QUERY_DESC_HDR_SIZE,
2096                                 UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
2097
2098                 /* replace non-printable or non-ASCII characters with spaces */
2099                 for (i = 0; i < ascii_len; i++)
2100                         ufshcd_remove_non_printable(&buff_ascii[i]);
2101
2102                 memset(buf + QUERY_DESC_HDR_SIZE, 0,
2103                                 size - QUERY_DESC_HDR_SIZE);
2104                 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
2105                 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
2106 out_free_buff:
2107                 kfree(buff_ascii);
2108         }
2109 out:
2110         return err;
2111 }
2112 EXPORT_SYMBOL(ufshcd_read_string_desc);
2113
2114 /**
2115  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
2116  * @hba: Pointer to adapter instance
2117  * @lun: lun id
2118  * @param_offset: offset of the parameter to read
2119  * @param_read_buf: pointer to buffer where parameter would be read
2120  * @param_size: sizeof(param_read_buf)
2121  *
2122  * Return 0 in case of success, non-zero otherwise
2123  */
2124 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
2125                                               int lun,
2126                                               enum unit_desc_param param_offset,
2127                                               u8 *param_read_buf,
2128                                               u32 param_size)
2129 {
2130         /*
2131          * Unit descriptors are only available for general purpose LUs (LUN id
2132          * from 0 to 7) and RPMB Well known LU.
2133          */
2134         if (lun != UFS_UPIU_RPMB_WLUN && (lun >= UFS_UPIU_MAX_GENERAL_LUN))
2135                 return -EOPNOTSUPP;
2136
2137         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
2138                                       param_offset, param_read_buf, param_size);
2139 }
2140
2141 /**
2142  * ufshcd_memory_alloc - allocate memory for host memory space data structures
2143  * @hba: per adapter instance
2144  *
2145  * 1. Allocate DMA memory for Command Descriptor array
2146  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
2147  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
2148  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
2149  *      (UTMRDL)
2150  * 4. Allocate memory for local reference block(lrb).
2151  *
2152  * Returns 0 for success, non-zero in case of failure
2153  */
2154 static int ufshcd_memory_alloc(struct ufs_hba *hba)
2155 {
2156         size_t utmrdl_size, utrdl_size, ucdl_size;
2157
2158         /* Allocate memory for UTP command descriptors */
2159         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
2160         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
2161                                                   ucdl_size,
2162                                                   &hba->ucdl_dma_addr,
2163                                                   GFP_KERNEL);
2164
2165         /*
2166          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
2167          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
2168          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
2169          * be aligned to 128 bytes as well
2170          */
2171         if (!hba->ucdl_base_addr ||
2172             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
2173                 dev_err(hba->dev,
2174                         "Command Descriptor Memory allocation failed\n");
2175                 goto out;
2176         }
2177
2178         /*
2179          * Allocate memory for UTP Transfer descriptors
2180          * UFSHCI requires 1024 byte alignment of UTRD
2181          */
2182         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
2183         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
2184                                                    utrdl_size,
2185                                                    &hba->utrdl_dma_addr,
2186                                                    GFP_KERNEL);
2187         if (!hba->utrdl_base_addr ||
2188             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
2189                 dev_err(hba->dev,
2190                         "Transfer Descriptor Memory allocation failed\n");
2191                 goto out;
2192         }
2193
2194         /*
2195          * Allocate memory for UTP Task Management descriptors
2196          * UFSHCI requires 1024 byte alignment of UTMRD
2197          */
2198         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
2199         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
2200                                                     utmrdl_size,
2201                                                     &hba->utmrdl_dma_addr,
2202                                                     GFP_KERNEL);
2203         if (!hba->utmrdl_base_addr ||
2204             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
2205                 dev_err(hba->dev,
2206                 "Task Management Descriptor Memory allocation failed\n");
2207                 goto out;
2208         }
2209
2210         /* Allocate memory for local reference block */
2211         hba->lrb = devm_kzalloc(hba->dev,
2212                                 hba->nutrs * sizeof(struct ufshcd_lrb),
2213                                 GFP_KERNEL);
2214         if (!hba->lrb) {
2215                 dev_err(hba->dev, "LRB Memory allocation failed\n");
2216                 goto out;
2217         }
2218         return 0;
2219 out:
2220         return -ENOMEM;
2221 }
2222
2223 /**
2224  * ufshcd_host_memory_configure - configure local reference block with
2225  *                              memory offsets
2226  * @hba: per adapter instance
2227  *
2228  * Configure Host memory space
2229  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
2230  * address.
2231  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
2232  * and PRDT offset.
2233  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
2234  * into local reference block.
2235  */
2236 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
2237 {
2238         struct utp_transfer_cmd_desc *cmd_descp;
2239         struct utp_transfer_req_desc *utrdlp;
2240         dma_addr_t cmd_desc_dma_addr;
2241         dma_addr_t cmd_desc_element_addr;
2242         u16 response_offset;
2243         u16 prdt_offset;
2244         int cmd_desc_size;
2245         int i;
2246
2247         utrdlp = hba->utrdl_base_addr;
2248         cmd_descp = hba->ucdl_base_addr;
2249
2250         response_offset =
2251                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
2252         prdt_offset =
2253                 offsetof(struct utp_transfer_cmd_desc, prd_table);
2254
2255         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
2256         cmd_desc_dma_addr = hba->ucdl_dma_addr;
2257
2258         for (i = 0; i < hba->nutrs; i++) {
2259                 /* Configure UTRD with command descriptor base address */
2260                 cmd_desc_element_addr =
2261                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
2262                 utrdlp[i].command_desc_base_addr_lo =
2263                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
2264                 utrdlp[i].command_desc_base_addr_hi =
2265                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
2266
2267                 /* Response upiu and prdt offset should be in double words */
2268                 utrdlp[i].response_upiu_offset =
2269                                 cpu_to_le16((response_offset >> 2));
2270                 utrdlp[i].prd_table_offset =
2271                                 cpu_to_le16((prdt_offset >> 2));
2272                 utrdlp[i].response_upiu_length =
2273                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
2274
2275                 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
2276                 hba->lrb[i].ucd_req_ptr =
2277                         (struct utp_upiu_req *)(cmd_descp + i);
2278                 hba->lrb[i].ucd_rsp_ptr =
2279                         (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2280                 hba->lrb[i].ucd_prdt_ptr =
2281                         (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2282         }
2283 }
2284
2285 /**
2286  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
2287  * @hba: per adapter instance
2288  *
2289  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
2290  * in order to initialize the Unipro link startup procedure.
2291  * Once the Unipro links are up, the device connected to the controller
2292  * is detected.
2293  *
2294  * Returns 0 on success, non-zero value on failure
2295  */
2296 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
2297 {
2298         struct uic_command uic_cmd = {0};
2299         int ret;
2300
2301         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
2302
2303         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2304         if (ret)
2305                 dev_err(hba->dev,
2306                         "dme-link-startup: error code %d\n", ret);
2307         return ret;
2308 }
2309
2310 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
2311 {
2312         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
2313         unsigned long min_sleep_time_us;
2314
2315         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
2316                 return;
2317
2318         /*
2319          * last_dme_cmd_tstamp will be 0 only for 1st call to
2320          * this function
2321          */
2322         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
2323                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
2324         } else {
2325                 unsigned long delta =
2326                         (unsigned long) ktime_to_us(
2327                                 ktime_sub(ktime_get(),
2328                                 hba->last_dme_cmd_tstamp));
2329
2330                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
2331                         min_sleep_time_us =
2332                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
2333                 else
2334                         return; /* no more delay required */
2335         }
2336
2337         /* allow sleep for extra 50us if needed */
2338         usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
2339 }
2340
2341 /**
2342  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
2343  * @hba: per adapter instance
2344  * @attr_sel: uic command argument1
2345  * @attr_set: attribute set type as uic command argument2
2346  * @mib_val: setting value as uic command argument3
2347  * @peer: indicate whether peer or local
2348  *
2349  * Returns 0 on success, non-zero value on failure
2350  */
2351 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
2352                         u8 attr_set, u32 mib_val, u8 peer)
2353 {
2354         struct uic_command uic_cmd = {0};
2355         static const char *const action[] = {
2356                 "dme-set",
2357                 "dme-peer-set"
2358         };
2359         const char *set = action[!!peer];
2360         int ret;
2361         int retries = UFS_UIC_COMMAND_RETRIES;
2362
2363         uic_cmd.command = peer ?
2364                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
2365         uic_cmd.argument1 = attr_sel;
2366         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
2367         uic_cmd.argument3 = mib_val;
2368
2369         do {
2370                 /* for peer attributes we retry upon failure */
2371                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2372                 if (ret)
2373                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
2374                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
2375         } while (ret && peer && --retries);
2376
2377         if (!retries)
2378                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
2379                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val,
2380                                 retries);
2381
2382         return ret;
2383 }
2384 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
2385
2386 /**
2387  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
2388  * @hba: per adapter instance
2389  * @attr_sel: uic command argument1
2390  * @mib_val: the value of the attribute as returned by the UIC command
2391  * @peer: indicate whether peer or local
2392  *
2393  * Returns 0 on success, non-zero value on failure
2394  */
2395 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
2396                         u32 *mib_val, u8 peer)
2397 {
2398         struct uic_command uic_cmd = {0};
2399         static const char *const action[] = {
2400                 "dme-get",
2401                 "dme-peer-get"
2402         };
2403         const char *get = action[!!peer];
2404         int ret;
2405         int retries = UFS_UIC_COMMAND_RETRIES;
2406         struct ufs_pa_layer_attr orig_pwr_info;
2407         struct ufs_pa_layer_attr temp_pwr_info;
2408         bool pwr_mode_change = false;
2409
2410         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
2411                 orig_pwr_info = hba->pwr_info;
2412                 temp_pwr_info = orig_pwr_info;
2413
2414                 if (orig_pwr_info.pwr_tx == FAST_MODE ||
2415                     orig_pwr_info.pwr_rx == FAST_MODE) {
2416                         temp_pwr_info.pwr_tx = FASTAUTO_MODE;
2417                         temp_pwr_info.pwr_rx = FASTAUTO_MODE;
2418                         pwr_mode_change = true;
2419                 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
2420                     orig_pwr_info.pwr_rx == SLOW_MODE) {
2421                         temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
2422                         temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
2423                         pwr_mode_change = true;
2424                 }
2425                 if (pwr_mode_change) {
2426                         ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
2427                         if (ret)
2428                                 goto out;
2429                 }
2430         }
2431
2432         uic_cmd.command = peer ?
2433                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
2434         uic_cmd.argument1 = attr_sel;
2435
2436         do {
2437                 /* for peer attributes we retry upon failure */
2438                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
2439                 if (ret)
2440                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
2441                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
2442         } while (ret && peer && --retries);
2443
2444         if (!retries)
2445                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
2446                                 get, UIC_GET_ATTR_ID(attr_sel), retries);
2447
2448         if (mib_val && !ret)
2449                 *mib_val = uic_cmd.argument3;
2450
2451         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
2452             && pwr_mode_change)
2453                 ufshcd_change_power_mode(hba, &orig_pwr_info);
2454 out:
2455         return ret;
2456 }
2457 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
2458
2459 /**
2460  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
2461  * state) and waits for it to take effect.
2462  *
2463  * @hba: per adapter instance
2464  * @cmd: UIC command to execute
2465  *
2466  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
2467  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
2468  * and device UniPro link and hence it's final completion would be indicated by
2469  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
2470  * addition to normal UIC command completion Status (UCCS). This function only
2471  * returns after the relevant status bits indicate the completion.
2472  *
2473  * Returns 0 on success, non-zero value on failure
2474  */
2475 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
2476 {
2477         struct completion uic_async_done;
2478         unsigned long flags;
2479         u8 status;
2480         int ret;
2481         bool reenable_intr = false;
2482
2483         mutex_lock(&hba->uic_cmd_mutex);
2484         init_completion(&uic_async_done);
2485         ufshcd_add_delay_before_dme_cmd(hba);
2486
2487         spin_lock_irqsave(hba->host->host_lock, flags);
2488         hba->uic_async_done = &uic_async_done;
2489         if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
2490                 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
2491                 /*
2492                  * Make sure UIC command completion interrupt is disabled before
2493                  * issuing UIC command.
2494                  */
2495                 wmb();
2496                 reenable_intr = true;
2497         }
2498         ret = __ufshcd_send_uic_cmd(hba, cmd, false);
2499         spin_unlock_irqrestore(hba->host->host_lock, flags);
2500         if (ret) {
2501                 dev_err(hba->dev,
2502                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
2503                         cmd->command, cmd->argument3, ret);
2504                 goto out;
2505         }
2506
2507         if (!wait_for_completion_timeout(hba->uic_async_done,
2508                                          msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2509                 dev_err(hba->dev,
2510                         "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
2511                         cmd->command, cmd->argument3);
2512                 ret = -ETIMEDOUT;
2513                 goto out;
2514         }
2515
2516         status = ufshcd_get_upmcrs(hba);
2517         if (status != PWR_LOCAL) {
2518                 dev_err(hba->dev,
2519                         "pwr ctrl cmd 0x%0x failed, host umpcrs:0x%x\n",
2520                         cmd->command, status);
2521                 ret = (status != PWR_OK) ? status : -1;
2522         }
2523 out:
2524         spin_lock_irqsave(hba->host->host_lock, flags);
2525         hba->active_uic_cmd = NULL;
2526         hba->uic_async_done = NULL;
2527         if (reenable_intr)
2528                 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
2529         spin_unlock_irqrestore(hba->host->host_lock, flags);
2530         mutex_unlock(&hba->uic_cmd_mutex);
2531
2532         return ret;
2533 }
2534
2535 /**
2536  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
2537  *                              using DME_SET primitives.
2538  * @hba: per adapter instance
2539  * @mode: powr mode value
2540  *
2541  * Returns 0 on success, non-zero value on failure
2542  */
2543 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
2544 {
2545         struct uic_command uic_cmd = {0};
2546         int ret;
2547
2548         if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
2549                 ret = ufshcd_dme_set(hba,
2550                                 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
2551                 if (ret) {
2552                         dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
2553                                                 __func__, ret);
2554                         goto out;
2555                 }
2556         }
2557
2558         uic_cmd.command = UIC_CMD_DME_SET;
2559         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
2560         uic_cmd.argument3 = mode;
2561         ufshcd_hold(hba, false);
2562         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2563         ufshcd_release(hba);
2564
2565 out:
2566         return ret;
2567 }
2568
2569 static int ufshcd_link_recovery(struct ufs_hba *hba)
2570 {
2571         int ret;
2572         unsigned long flags;
2573
2574         spin_lock_irqsave(hba->host->host_lock, flags);
2575         hba->ufshcd_state = UFSHCD_STATE_RESET;
2576         ufshcd_set_eh_in_progress(hba);
2577         spin_unlock_irqrestore(hba->host->host_lock, flags);
2578
2579         ret = ufshcd_host_reset_and_restore(hba);
2580
2581         spin_lock_irqsave(hba->host->host_lock, flags);
2582         if (ret)
2583                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
2584         ufshcd_clear_eh_in_progress(hba);
2585         spin_unlock_irqrestore(hba->host->host_lock, flags);
2586
2587         if (ret)
2588                 dev_err(hba->dev, "%s: link recovery failed, err %d",
2589                         __func__, ret);
2590
2591         return ret;
2592 }
2593
2594 static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
2595 {
2596         int ret;
2597         struct uic_command uic_cmd = {0};
2598
2599         uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
2600         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2601
2602         if (ret) {
2603                 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
2604                         __func__, ret);
2605
2606                 /*
2607                  * If link recovery fails then return error so that caller
2608                  * don't retry the hibern8 enter again.
2609                  */
2610                 if (ufshcd_link_recovery(hba))
2611                         ret = -ENOLINK;
2612         }
2613
2614         return ret;
2615 }
2616
2617 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
2618 {
2619         int ret = 0, retries;
2620
2621         for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
2622                 ret = __ufshcd_uic_hibern8_enter(hba);
2623                 if (!ret || ret == -ENOLINK)
2624                         goto out;
2625         }
2626 out:
2627         return ret;
2628 }
2629
2630 static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
2631 {
2632         struct uic_command uic_cmd = {0};
2633         int ret;
2634
2635         uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
2636         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
2637         if (ret) {
2638                 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
2639                         __func__, ret);
2640                 ret = ufshcd_link_recovery(hba);
2641         }
2642
2643         return ret;
2644 }
2645
2646  /**
2647  * ufshcd_init_pwr_info - setting the POR (power on reset)
2648  * values in hba power info
2649  * @hba: per-adapter instance
2650  */
2651 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
2652 {
2653         hba->pwr_info.gear_rx = UFS_PWM_G1;
2654         hba->pwr_info.gear_tx = UFS_PWM_G1;
2655         hba->pwr_info.lane_rx = 1;
2656         hba->pwr_info.lane_tx = 1;
2657         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
2658         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
2659         hba->pwr_info.hs_rate = 0;
2660 }
2661
2662 /**
2663  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
2664  * @hba: per-adapter instance
2665  */
2666 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
2667 {
2668         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
2669
2670         if (hba->max_pwr_info.is_valid)
2671                 return 0;
2672
2673         pwr_info->pwr_tx = FASTAUTO_MODE;
2674         pwr_info->pwr_rx = FASTAUTO_MODE;
2675         pwr_info->hs_rate = PA_HS_MODE_B;
2676
2677         /* Get the connected lane count */
2678         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
2679                         &pwr_info->lane_rx);
2680         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
2681                         &pwr_info->lane_tx);
2682
2683         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
2684                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
2685                                 __func__,
2686                                 pwr_info->lane_rx,
2687                                 pwr_info->lane_tx);
2688                 return -EINVAL;
2689         }
2690
2691         /*
2692          * First, get the maximum gears of HS speed.
2693          * If a zero value, it means there is no HSGEAR capability.
2694          * Then, get the maximum gears of PWM speed.
2695          */
2696         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
2697         if (!pwr_info->gear_rx) {
2698                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
2699                                 &pwr_info->gear_rx);
2700                 if (!pwr_info->gear_rx) {
2701                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
2702                                 __func__, pwr_info->gear_rx);
2703                         return -EINVAL;
2704                 }
2705                 pwr_info->pwr_rx = SLOWAUTO_MODE;
2706         }
2707
2708         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
2709                         &pwr_info->gear_tx);
2710         if (!pwr_info->gear_tx) {
2711                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
2712                                 &pwr_info->gear_tx);
2713                 if (!pwr_info->gear_tx) {
2714                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
2715                                 __func__, pwr_info->gear_tx);
2716                         return -EINVAL;
2717                 }
2718                 pwr_info->pwr_tx = SLOWAUTO_MODE;
2719         }
2720
2721         hba->max_pwr_info.is_valid = true;
2722         return 0;
2723 }
2724
2725 static int ufshcd_change_power_mode(struct ufs_hba *hba,
2726                              struct ufs_pa_layer_attr *pwr_mode)
2727 {
2728         int ret;
2729
2730         /* if already configured to the requested pwr_mode */
2731         if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
2732             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
2733             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
2734             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
2735             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
2736             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
2737             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
2738                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
2739                 return 0;
2740         }
2741
2742         /*
2743          * Configure attributes for power mode change with below.
2744          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
2745          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
2746          * - PA_HSSERIES
2747          */
2748         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
2749         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
2750                         pwr_mode->lane_rx);
2751         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
2752                         pwr_mode->pwr_rx == FAST_MODE)
2753                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
2754         else
2755                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
2756
2757         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
2758         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
2759                         pwr_mode->lane_tx);
2760         if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
2761                         pwr_mode->pwr_tx == FAST_MODE)
2762                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
2763         else
2764                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
2765
2766         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
2767             pwr_mode->pwr_tx == FASTAUTO_MODE ||
2768             pwr_mode->pwr_rx == FAST_MODE ||
2769             pwr_mode->pwr_tx == FAST_MODE)
2770                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
2771                                                 pwr_mode->hs_rate);
2772
2773         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
2774                         | pwr_mode->pwr_tx);
2775
2776         if (ret) {
2777                 dev_err(hba->dev,
2778                         "%s: power mode change failed %d\n", __func__, ret);
2779         } else {
2780                 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
2781                                                                 pwr_mode);
2782
2783                 memcpy(&hba->pwr_info, pwr_mode,
2784                         sizeof(struct ufs_pa_layer_attr));
2785         }
2786
2787         return ret;
2788 }
2789
2790 /**
2791  * ufshcd_config_pwr_mode - configure a new power mode
2792  * @hba: per-adapter instance
2793  * @desired_pwr_mode: desired power configuration
2794  */
2795 static int ufshcd_config_pwr_mode(struct ufs_hba *hba,
2796                 struct ufs_pa_layer_attr *desired_pwr_mode)
2797 {
2798         struct ufs_pa_layer_attr final_params = { 0 };
2799         int ret;
2800
2801         ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
2802                                         desired_pwr_mode, &final_params);
2803
2804         if (ret)
2805                 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
2806
2807         ret = ufshcd_change_power_mode(hba, &final_params);
2808
2809         return ret;
2810 }
2811
2812 /**
2813  * ufshcd_complete_dev_init() - checks device readiness
2814  * hba: per-adapter instance
2815  *
2816  * Set fDeviceInit flag and poll until device toggles it.
2817  */
2818 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
2819 {
2820         int i;
2821         int err;
2822         bool flag_res = 1;
2823
2824         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
2825                 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
2826         if (err) {
2827                 dev_err(hba->dev,
2828                         "%s setting fDeviceInit flag failed with error %d\n",
2829                         __func__, err);
2830                 goto out;
2831         }
2832
2833         /* poll for max. 1000 iterations for fDeviceInit flag to clear */
2834         for (i = 0; i < 1000 && !err && flag_res; i++)
2835                 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
2836                         QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
2837
2838         if (err)
2839                 dev_err(hba->dev,
2840                         "%s reading fDeviceInit flag failed with error %d\n",
2841                         __func__, err);
2842         else if (flag_res)
2843                 dev_err(hba->dev,
2844                         "%s fDeviceInit was not cleared by the device\n",
2845                         __func__);
2846
2847 out:
2848         return err;
2849 }
2850
2851 /**
2852  * ufshcd_make_hba_operational - Make UFS controller operational
2853  * @hba: per adapter instance
2854  *
2855  * To bring UFS host controller to operational state,
2856  * 1. Enable required interrupts
2857  * 2. Configure interrupt aggregation
2858  * 3. Program UTRL and UTMRL base address
2859  * 4. Configure run-stop-registers
2860  *
2861  * Returns 0 on success, non-zero value on failure
2862  */
2863 static int ufshcd_make_hba_operational(struct ufs_hba *hba)
2864 {
2865         int err = 0;
2866         u32 reg;
2867
2868         /* Enable required interrupts */
2869         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
2870
2871         /* Configure interrupt aggregation */
2872         if (ufshcd_is_intr_aggr_allowed(hba))
2873                 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
2874         else
2875                 ufshcd_disable_intr_aggr(hba);
2876
2877         /* Configure UTRL and UTMRL base address registers */
2878         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
2879                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
2880         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
2881                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
2882         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
2883                         REG_UTP_TASK_REQ_LIST_BASE_L);
2884         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
2885                         REG_UTP_TASK_REQ_LIST_BASE_H);
2886
2887         /*
2888          * Make sure base address and interrupt setup are updated before
2889          * enabling the run/stop registers below.
2890          */
2891         wmb();
2892
2893         /*
2894          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
2895          */
2896         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
2897         if (!(ufshcd_get_lists_status(reg))) {
2898                 ufshcd_enable_run_stop_reg(hba);
2899         } else {
2900                 dev_err(hba->dev,
2901                         "Host controller not ready to process requests");
2902                 err = -EIO;
2903                 goto out;
2904         }
2905
2906 out:
2907         return err;
2908 }
2909
2910 /**
2911  * ufshcd_hba_stop - Send controller to reset state
2912  * @hba: per adapter instance
2913  * @can_sleep: perform sleep or just spin
2914  */
2915 static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
2916 {
2917         int err;
2918
2919         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
2920         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
2921                                         CONTROLLER_ENABLE, CONTROLLER_DISABLE,
2922                                         10, 1, can_sleep);
2923         if (err)
2924                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
2925 }
2926
2927 /**
2928  * ufshcd_hba_enable - initialize the controller
2929  * @hba: per adapter instance
2930  *
2931  * The controller resets itself and controller firmware initialization
2932  * sequence kicks off. When controller is ready it will set
2933  * the Host Controller Enable bit to 1.
2934  *
2935  * Returns 0 on success, non-zero value on failure
2936  */
2937 static int ufshcd_hba_enable(struct ufs_hba *hba)
2938 {
2939         int retry;
2940
2941         /*
2942          * msleep of 1 and 5 used in this function might result in msleep(20),
2943          * but it was necessary to send the UFS FPGA to reset mode during
2944          * development and testing of this driver. msleep can be changed to
2945          * mdelay and retry count can be reduced based on the controller.
2946          */
2947         if (!ufshcd_is_hba_active(hba))
2948                 /* change controller state to "reset state" */
2949                 ufshcd_hba_stop(hba, true);
2950
2951         /* UniPro link is disabled at this point */
2952         ufshcd_set_link_off(hba);
2953
2954         ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
2955
2956         /* start controller initialization sequence */
2957         ufshcd_hba_start(hba);
2958
2959         /*
2960          * To initialize a UFS host controller HCE bit must be set to 1.
2961          * During initialization the HCE bit value changes from 1->0->1.
2962          * When the host controller completes initialization sequence
2963          * it sets the value of HCE bit to 1. The same HCE bit is read back
2964          * to check if the controller has completed initialization sequence.
2965          * So without this delay the value HCE = 1, set in the previous
2966          * instruction might be read back.
2967          * This delay can be changed based on the controller.
2968          */
2969         msleep(1);
2970
2971         /* wait for the host controller to complete initialization */
2972         retry = 10;
2973         while (ufshcd_is_hba_active(hba)) {
2974                 if (retry) {
2975                         retry--;
2976                 } else {
2977                         dev_err(hba->dev,
2978                                 "Controller enable failed\n");
2979                         return -EIO;
2980                 }
2981                 msleep(5);
2982         }
2983
2984         /* enable UIC related interrupts */
2985         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
2986
2987         ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
2988
2989         return 0;
2990 }
2991
2992 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
2993 {
2994         int tx_lanes, i, err = 0;
2995
2996         if (!peer)
2997                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
2998                                &tx_lanes);
2999         else
3000                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
3001                                     &tx_lanes);
3002         for (i = 0; i < tx_lanes; i++) {
3003                 if (!peer)
3004                         err = ufshcd_dme_set(hba,
3005                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
3006                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
3007                                         0);
3008                 else
3009                         err = ufshcd_dme_peer_set(hba,
3010                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
3011                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
3012                                         0);
3013                 if (err) {
3014                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
3015                                 __func__, peer, i, err);
3016                         break;
3017                 }
3018         }
3019
3020         return err;
3021 }
3022
3023 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
3024 {
3025         return ufshcd_disable_tx_lcc(hba, true);
3026 }
3027
3028 /**
3029  * ufshcd_link_startup - Initialize unipro link startup
3030  * @hba: per adapter instance
3031  *
3032  * Returns 0 for success, non-zero in case of failure
3033  */
3034 static int ufshcd_link_startup(struct ufs_hba *hba)
3035 {
3036         int ret;
3037         int retries = DME_LINKSTARTUP_RETRIES;
3038
3039         do {
3040                 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
3041
3042                 ret = ufshcd_dme_link_startup(hba);
3043
3044                 /* check if device is detected by inter-connect layer */
3045                 if (!ret && !ufshcd_is_device_present(hba)) {
3046                         dev_err(hba->dev, "%s: Device not present\n", __func__);
3047                         ret = -ENXIO;
3048                         goto out;
3049                 }
3050
3051                 /*
3052                  * DME link lost indication is only received when link is up,
3053                  * but we can't be sure if the link is up until link startup
3054                  * succeeds. So reset the local Uni-Pro and try again.
3055                  */
3056                 if (ret && ufshcd_hba_enable(hba))
3057                         goto out;
3058         } while (ret && retries--);
3059
3060         if (ret)
3061                 /* failed to get the link up... retire */
3062                 goto out;
3063
3064         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
3065                 ret = ufshcd_disable_device_tx_lcc(hba);
3066                 if (ret)
3067                         goto out;
3068         }
3069
3070         /* Include any host controller configuration via UIC commands */
3071         ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
3072         if (ret)
3073                 goto out;
3074
3075         ret = ufshcd_make_hba_operational(hba);
3076 out:
3077         if (ret)
3078                 dev_err(hba->dev, "link startup failed %d\n", ret);
3079         return ret;
3080 }
3081
3082 /**
3083  * ufshcd_verify_dev_init() - Verify device initialization
3084  * @hba: per-adapter instance
3085  *
3086  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
3087  * device Transport Protocol (UTP) layer is ready after a reset.
3088  * If the UTP layer at the device side is not initialized, it may
3089  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
3090  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
3091  */
3092 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
3093 {
3094         int err = 0;
3095         int retries;
3096
3097         ufshcd_hold(hba, false);
3098         mutex_lock(&hba->dev_cmd.lock);
3099         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
3100                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
3101                                                NOP_OUT_TIMEOUT);
3102
3103                 if (!err || err == -ETIMEDOUT)
3104                         break;
3105
3106                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
3107         }
3108         mutex_unlock(&hba->dev_cmd.lock);
3109         ufshcd_release(hba);
3110
3111         if (err)
3112                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
3113         return err;
3114 }
3115
3116 /**
3117  * ufshcd_set_queue_depth - set lun queue depth
3118  * @sdev: pointer to SCSI device
3119  *
3120  * Read bLUQueueDepth value and activate scsi tagged command
3121  * queueing. For WLUN, queue depth is set to 1. For best-effort
3122  * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
3123  * value that host can queue.
3124  */
3125 static void ufshcd_set_queue_depth(struct scsi_device *sdev)
3126 {
3127         int ret = 0;
3128         u8 lun_qdepth;
3129         struct ufs_hba *hba;
3130
3131         hba = shost_priv(sdev->host);
3132
3133         lun_qdepth = hba->nutrs;
3134         ret = ufshcd_read_unit_desc_param(hba,
3135                                           ufshcd_scsi_to_upiu_lun(sdev->lun),
3136                                           UNIT_DESC_PARAM_LU_Q_DEPTH,
3137                                           &lun_qdepth,
3138                                           sizeof(lun_qdepth));
3139
3140         /* Some WLUN doesn't support unit descriptor */
3141         if (ret == -EOPNOTSUPP)
3142                 lun_qdepth = 1;
3143         else if (!lun_qdepth)
3144                 /* eventually, we can figure out the real queue depth */
3145                 lun_qdepth = hba->nutrs;
3146         else
3147                 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
3148
3149         dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
3150                         __func__, lun_qdepth);
3151         scsi_change_queue_depth(sdev, lun_qdepth);
3152 }
3153
3154 /*
3155  * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
3156  * @hba: per-adapter instance
3157  * @lun: UFS device lun id
3158  * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
3159  *
3160  * Returns 0 in case of success and b_lu_write_protect status would be returned
3161  * @b_lu_write_protect parameter.
3162  * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
3163  * Returns -EINVAL in case of invalid parameters passed to this function.
3164  */
3165 static int ufshcd_get_lu_wp(struct ufs_hba *hba,
3166                             u8 lun,
3167                             u8 *b_lu_write_protect)
3168 {
3169         int ret;
3170
3171         if (!b_lu_write_protect)
3172                 ret = -EINVAL;
3173         /*
3174          * According to UFS device spec, RPMB LU can't be write
3175          * protected so skip reading bLUWriteProtect parameter for
3176          * it. For other W-LUs, UNIT DESCRIPTOR is not available.
3177          */
3178         else if (lun >= UFS_UPIU_MAX_GENERAL_LUN)
3179                 ret = -ENOTSUPP;
3180         else
3181                 ret = ufshcd_read_unit_desc_param(hba,
3182                                           lun,
3183                                           UNIT_DESC_PARAM_LU_WR_PROTECT,
3184                                           b_lu_write_protect,
3185                                           sizeof(*b_lu_write_protect));
3186         return ret;
3187 }
3188
3189 /**
3190  * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
3191  * status
3192  * @hba: per-adapter instance
3193  * @sdev: pointer to SCSI device
3194  *
3195  */
3196 static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
3197                                                     struct scsi_device *sdev)
3198 {
3199         if (hba->dev_info.f_power_on_wp_en &&
3200             !hba->dev_info.is_lu_power_on_wp) {
3201                 u8 b_lu_write_protect;
3202
3203                 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
3204                                       &b_lu_write_protect) &&
3205                     (b_lu_write_protect == UFS_LU_POWER_ON_WP))
3206                         hba->dev_info.is_lu_power_on_wp = true;
3207         }
3208 }
3209
3210 /**
3211  * ufshcd_slave_alloc - handle initial SCSI device configurations
3212  * @sdev: pointer to SCSI device
3213  *
3214  * Returns success
3215  */
3216 static int ufshcd_slave_alloc(struct scsi_device *sdev)
3217 {
3218         struct ufs_hba *hba;
3219
3220         hba = shost_priv(sdev->host);
3221
3222         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
3223         sdev->use_10_for_ms = 1;
3224
3225         /* allow SCSI layer to restart the device in case of errors */
3226         sdev->allow_restart = 1;
3227
3228         /* REPORT SUPPORTED OPERATION CODES is not supported */
3229         sdev->no_report_opcodes = 1;
3230
3231
3232         ufshcd_set_queue_depth(sdev);
3233
3234         ufshcd_get_lu_power_on_wp_status(hba, sdev);
3235
3236         return 0;
3237 }
3238
3239 /**
3240  * ufshcd_change_queue_depth - change queue depth
3241  * @sdev: pointer to SCSI device
3242  * @depth: required depth to set
3243  *
3244  * Change queue depth and make sure the max. limits are not crossed.
3245  */
3246 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
3247 {
3248         struct ufs_hba *hba = shost_priv(sdev->host);
3249
3250         if (depth > hba->nutrs)
3251                 depth = hba->nutrs;
3252         return scsi_change_queue_depth(sdev, depth);
3253 }
3254
3255 /**
3256  * ufshcd_slave_configure - adjust SCSI device configurations
3257  * @sdev: pointer to SCSI device
3258  */
3259 static int ufshcd_slave_configure(struct scsi_device *sdev)
3260 {
3261         struct request_queue *q = sdev->request_queue;
3262
3263         blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
3264         blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX);
3265
3266         return 0;
3267 }
3268
3269 /**
3270  * ufshcd_slave_destroy - remove SCSI device configurations
3271  * @sdev: pointer to SCSI device
3272  */
3273 static void ufshcd_slave_destroy(struct scsi_device *sdev)
3274 {
3275         struct ufs_hba *hba;
3276
3277         hba = shost_priv(sdev->host);
3278         /* Drop the reference as it won't be needed anymore */
3279         if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
3280                 unsigned long flags;
3281
3282                 spin_lock_irqsave(hba->host->host_lock, flags);
3283                 hba->sdev_ufs_device = NULL;
3284                 spin_unlock_irqrestore(hba->host->host_lock, flags);
3285         }
3286 }
3287
3288 /**
3289  * ufshcd_task_req_compl - handle task management request completion
3290  * @hba: per adapter instance
3291  * @index: index of the completed request
3292  * @resp: task management service response
3293  *
3294  * Returns non-zero value on error, zero on success
3295  */
3296 static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp)
3297 {
3298         struct utp_task_req_desc *task_req_descp;
3299         struct utp_upiu_task_rsp *task_rsp_upiup;
3300         unsigned long flags;
3301         int ocs_value;
3302         int task_result;
3303
3304         spin_lock_irqsave(hba->host->host_lock, flags);
3305
3306         /* Clear completed tasks from outstanding_tasks */
3307         __clear_bit(index, &hba->outstanding_tasks);
3308
3309         task_req_descp = hba->utmrdl_base_addr;
3310         ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
3311
3312         if (ocs_value == OCS_SUCCESS) {
3313                 task_rsp_upiup = (struct utp_upiu_task_rsp *)
3314                                 task_req_descp[index].task_rsp_upiu;
3315                 task_result = be32_to_cpu(task_rsp_upiup->header.dword_1);
3316                 task_result = ((task_result & MASK_TASK_RESPONSE) >> 8);
3317                 if (resp)
3318                         *resp = (u8)task_result;
3319         } else {
3320                 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
3321                                 __func__, ocs_value);
3322         }
3323         spin_unlock_irqrestore(hba->host->host_lock, flags);
3324
3325         return ocs_value;
3326 }
3327
3328 /**
3329  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
3330  * @lrb: pointer to local reference block of completed command
3331  * @scsi_status: SCSI command status
3332  *
3333  * Returns value base on SCSI command status
3334  */
3335 static inline int
3336 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
3337 {
3338         int result = 0;
3339
3340         switch (scsi_status) {
3341         case SAM_STAT_CHECK_CONDITION:
3342                 ufshcd_copy_sense_data(lrbp);
3343         case SAM_STAT_GOOD:
3344                 result |= DID_OK << 16 |
3345                           COMMAND_COMPLETE << 8 |
3346                           scsi_status;
3347                 break;
3348         case SAM_STAT_TASK_SET_FULL:
3349         case SAM_STAT_BUSY:
3350         case SAM_STAT_TASK_ABORTED:
3351                 ufshcd_copy_sense_data(lrbp);
3352                 result |= scsi_status;
3353                 break;
3354         default:
3355                 result |= DID_ERROR << 16;
3356                 break;
3357         } /* end of switch */
3358
3359         return result;
3360 }
3361
3362 /**
3363  * ufshcd_transfer_rsp_status - Get overall status of the response
3364  * @hba: per adapter instance
3365  * @lrb: pointer to local reference block of completed command
3366  *
3367  * Returns result of the command to notify SCSI midlayer
3368  */
3369 static inline int
3370 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
3371 {
3372         int result = 0;
3373         int scsi_status;
3374         int ocs;
3375
3376         /* overall command status of utrd */
3377         ocs = ufshcd_get_tr_ocs(lrbp);
3378
3379         switch (ocs) {
3380         case OCS_SUCCESS:
3381                 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3382
3383                 switch (result) {
3384                 case UPIU_TRANSACTION_RESPONSE:
3385                         /*
3386                          * get the response UPIU result to extract
3387                          * the SCSI command status
3388                          */
3389                         result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
3390
3391                         /*
3392                          * get the result based on SCSI status response
3393                          * to notify the SCSI midlayer of the command status
3394                          */
3395                         scsi_status = result & MASK_SCSI_STATUS;
3396                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
3397
3398                         /*
3399                          * Currently we are only supporting BKOPs exception
3400                          * events hence we can ignore BKOPs exception event
3401                          * during power management callbacks. BKOPs exception
3402                          * event is not expected to be raised in runtime suspend
3403                          * callback as it allows the urgent bkops.
3404                          * During system suspend, we are anyway forcefully
3405                          * disabling the bkops and if urgent bkops is needed
3406                          * it will be enabled on system resume. Long term
3407                          * solution could be to abort the system suspend if
3408                          * UFS device needs urgent BKOPs.
3409                          */
3410                         if (!hba->pm_op_in_progress &&
3411                             ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
3412                                 schedule_work(&hba->eeh_work);
3413                         break;
3414                 case UPIU_TRANSACTION_REJECT_UPIU:
3415                         /* TODO: handle Reject UPIU Response */
3416                         result = DID_ERROR << 16;
3417                         dev_err(hba->dev,
3418                                 "Reject UPIU not fully implemented\n");
3419                         break;
3420                 default:
3421                         result = DID_ERROR << 16;
3422                         dev_err(hba->dev,
3423                                 "Unexpected request response code = %x\n",
3424                                 result);
3425                         break;
3426                 }
3427                 break;
3428         case OCS_ABORTED:
3429                 result |= DID_ABORT << 16;
3430                 break;
3431         case OCS_INVALID_COMMAND_STATUS:
3432                 result |= DID_REQUEUE << 16;
3433                 break;
3434         case OCS_INVALID_CMD_TABLE_ATTR:
3435         case OCS_INVALID_PRDT_ATTR:
3436         case OCS_MISMATCH_DATA_BUF_SIZE:
3437         case OCS_MISMATCH_RESP_UPIU_SIZE:
3438         case OCS_PEER_COMM_FAILURE:
3439         case OCS_FATAL_ERROR:
3440         default:
3441                 result |= DID_ERROR << 16;
3442                 dev_err(hba->dev,
3443                 "OCS error from controller = %x\n", ocs);
3444                 break;
3445         } /* end of switch */
3446
3447         return result;
3448 }
3449
3450 /**
3451  * ufshcd_uic_cmd_compl - handle completion of uic command
3452  * @hba: per adapter instance
3453  * @intr_status: interrupt status generated by the controller
3454  */
3455 static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
3456 {
3457         if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
3458                 hba->active_uic_cmd->argument2 |=
3459                         ufshcd_get_uic_cmd_result(hba);
3460                 hba->active_uic_cmd->argument3 =
3461                         ufshcd_get_dme_attr_val(hba);
3462                 complete(&hba->active_uic_cmd->done);
3463         }
3464
3465         if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done)
3466                 complete(hba->uic_async_done);
3467 }
3468
3469 /**
3470  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
3471  * @hba: per adapter instance
3472  * @completed_reqs: requests to complete
3473  */
3474 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
3475                                         unsigned long completed_reqs)
3476 {
3477         struct ufshcd_lrb *lrbp;
3478         struct scsi_cmnd *cmd;
3479         int result;
3480         int index;
3481
3482         for_each_set_bit(index, &completed_reqs, hba->nutrs) {
3483                 lrbp = &hba->lrb[index];
3484                 cmd = lrbp->cmd;
3485                 if (cmd) {
3486                         result = ufshcd_transfer_rsp_status(hba, lrbp);
3487                         scsi_dma_unmap(cmd);
3488                         cmd->result = result;
3489                         /* Mark completed command as NULL in LRB */
3490                         lrbp->cmd = NULL;
3491                         clear_bit_unlock(index, &hba->lrb_in_use);
3492                         /* Do not touch lrbp after scsi done */
3493                         cmd->scsi_done(cmd);
3494                         __ufshcd_release(hba);
3495                 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE) {
3496                         if (hba->dev_cmd.complete)
3497                                 complete(hba->dev_cmd.complete);
3498                 }
3499         }
3500
3501         /* clear corresponding bits of completed commands */
3502         hba->outstanding_reqs ^= completed_reqs;
3503
3504         ufshcd_clk_scaling_update_busy(hba);
3505
3506         /* we might have free'd some tags above */
3507         wake_up(&hba->dev_cmd.tag_wq);
3508 }
3509
3510 /**
3511  * ufshcd_transfer_req_compl - handle SCSI and query command completion
3512  * @hba: per adapter instance
3513  */
3514 static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
3515 {
3516         unsigned long completed_reqs;
3517         u32 tr_doorbell;
3518
3519         /* Resetting interrupt aggregation counters first and reading the
3520          * DOOR_BELL afterward allows us to handle all the completed requests.
3521          * In order to prevent other interrupts starvation the DB is read once
3522          * after reset. The down side of this solution is the possibility of
3523          * false interrupt if device completes another request after resetting
3524          * aggregation and before reading the DB.
3525          */
3526         if (ufshcd_is_intr_aggr_allowed(hba))
3527                 ufshcd_reset_intr_aggr(hba);
3528
3529         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
3530         completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
3531
3532         __ufshcd_transfer_req_compl(hba, completed_reqs);
3533 }
3534
3535 /**
3536  * ufshcd_disable_ee - disable exception event
3537  * @hba: per-adapter instance
3538  * @mask: exception event to disable
3539  *
3540  * Disables exception event in the device so that the EVENT_ALERT
3541  * bit is not set.
3542  *
3543  * Returns zero on success, non-zero error value on failure.
3544  */
3545 static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
3546 {
3547         int err = 0;
3548         u32 val;
3549
3550         if (!(hba->ee_ctrl_mask & mask))
3551                 goto out;
3552
3553         val = hba->ee_ctrl_mask & ~mask;
3554         val &= 0xFFFF; /* 2 bytes */
3555         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
3556                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
3557         if (!err)
3558                 hba->ee_ctrl_mask &= ~mask;
3559 out:
3560         return err;
3561 }
3562
3563 /**
3564  * ufshcd_enable_ee - enable exception event
3565  * @hba: per-adapter instance
3566  * @mask: exception event to enable
3567  *
3568  * Enable corresponding exception event in the device to allow
3569  * device to alert host in critical scenarios.
3570  *
3571  * Returns zero on success, non-zero error value on failure.
3572  */
3573 static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
3574 {
3575         int err = 0;
3576         u32 val;
3577
3578         if (hba->ee_ctrl_mask & mask)
3579                 goto out;
3580
3581         val = hba->ee_ctrl_mask | mask;
3582         val &= 0xFFFF; /* 2 bytes */
3583         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
3584                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
3585         if (!err)
3586                 hba->ee_ctrl_mask |= mask;
3587 out:
3588         return err;
3589 }
3590
3591 /**
3592  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
3593  * @hba: per-adapter instance
3594  *
3595  * Allow device to manage background operations on its own. Enabling
3596  * this might lead to inconsistent latencies during normal data transfers
3597  * as the device is allowed to manage its own way of handling background
3598  * operations.
3599  *
3600  * Returns zero on success, non-zero on failure.
3601  */
3602 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
3603 {
3604         int err = 0;
3605
3606         if (hba->auto_bkops_enabled)
3607                 goto out;
3608
3609         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
3610                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
3611         if (err) {
3612                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
3613                                 __func__, err);
3614                 goto out;
3615         }
3616
3617         hba->auto_bkops_enabled = true;
3618
3619         /* No need of URGENT_BKOPS exception from the device */
3620         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
3621         if (err)
3622                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
3623                                 __func__, err);
3624 out:
3625         return err;
3626 }
3627
3628 /**
3629  * ufshcd_disable_auto_bkops - block device in doing background operations
3630  * @hba: per-adapter instance
3631  *
3632  * Disabling background operations improves command response latency but
3633  * has drawback of device moving into critical state where the device is
3634  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
3635  * host is idle so that BKOPS are managed effectively without any negative
3636  * impacts.
3637  *
3638  * Returns zero on success, non-zero on failure.
3639  */
3640 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
3641 {
3642         int err = 0;
3643
3644         if (!hba->auto_bkops_enabled)
3645                 goto out;
3646
3647         /*
3648          * If host assisted BKOPs is to be enabled, make sure
3649          * urgent bkops exception is allowed.
3650          */
3651         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
3652         if (err) {
3653                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
3654                                 __func__, err);
3655                 goto out;
3656         }
3657
3658         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
3659                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
3660         if (err) {
3661                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
3662                                 __func__, err);
3663                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
3664                 goto out;
3665         }
3666
3667         hba->auto_bkops_enabled = false;
3668 out:
3669         return err;
3670 }
3671
3672 /**
3673  * ufshcd_force_reset_auto_bkops - force enable of auto bkops
3674  * @hba: per adapter instance
3675  *
3676  * After a device reset the device may toggle the BKOPS_EN flag
3677  * to default value. The s/w tracking variables should be updated
3678  * as well. Do this by forcing enable of auto bkops.
3679  */
3680 static void  ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
3681 {
3682         hba->auto_bkops_enabled = false;
3683         hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
3684         ufshcd_enable_auto_bkops(hba);
3685 }
3686
3687 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
3688 {
3689         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3690                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
3691 }
3692
3693 /**
3694  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
3695  * @hba: per-adapter instance
3696  * @status: bkops_status value
3697  *
3698  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
3699  * flag in the device to permit background operations if the device
3700  * bkops_status is greater than or equal to "status" argument passed to
3701  * this function, disable otherwise.
3702  *
3703  * Returns 0 for success, non-zero in case of failure.
3704  *
3705  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
3706  * to know whether auto bkops is enabled or disabled after this function
3707  * returns control to it.
3708  */
3709 static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
3710                              enum bkops_status status)
3711 {
3712         int err;
3713         u32 curr_status = 0;
3714
3715         err = ufshcd_get_bkops_status(hba, &curr_status);
3716         if (err) {
3717                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
3718                                 __func__, err);
3719                 goto out;
3720         } else if (curr_status > BKOPS_STATUS_MAX) {
3721                 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
3722                                 __func__, curr_status);
3723                 err = -EINVAL;
3724                 goto out;
3725         }
3726
3727         if (curr_status >= status)
3728                 err = ufshcd_enable_auto_bkops(hba);
3729         else
3730                 err = ufshcd_disable_auto_bkops(hba);
3731 out:
3732         return err;
3733 }
3734
3735 /**
3736  * ufshcd_urgent_bkops - handle urgent bkops exception event
3737  * @hba: per-adapter instance
3738  *
3739  * Enable fBackgroundOpsEn flag in the device to permit background
3740  * operations.
3741  *
3742  * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
3743  * and negative error value for any other failure.
3744  */
3745 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
3746 {
3747         return ufshcd_bkops_ctrl(hba, BKOPS_STATUS_PERF_IMPACT);
3748 }
3749
3750 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
3751 {
3752         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3753                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
3754 }
3755
3756 /**
3757  * ufshcd_exception_event_handler - handle exceptions raised by device
3758  * @work: pointer to work data
3759  *
3760  * Read bExceptionEventStatus attribute from the device and handle the
3761  * exception event accordingly.
3762  */
3763 static void ufshcd_exception_event_handler(struct work_struct *work)
3764 {
3765         struct ufs_hba *hba;
3766         int err;
3767         u32 status = 0;
3768         hba = container_of(work, struct ufs_hba, eeh_work);
3769
3770         pm_runtime_get_sync(hba->dev);
3771         err = ufshcd_get_ee_status(hba, &status);
3772         if (err) {
3773                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
3774                                 __func__, err);
3775                 goto out;
3776         }
3777
3778         status &= hba->ee_ctrl_mask;
3779         if (status & MASK_EE_URGENT_BKOPS) {
3780                 err = ufshcd_urgent_bkops(hba);
3781                 if (err < 0)
3782                         dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
3783                                         __func__, err);
3784         }
3785 out:
3786         pm_runtime_put_sync(hba->dev);
3787         return;
3788 }
3789
3790 /* Complete requests that have door-bell cleared */
3791 static void ufshcd_complete_requests(struct ufs_hba *hba)
3792 {
3793         ufshcd_transfer_req_compl(hba);
3794         ufshcd_tmc_handler(hba);
3795 }
3796
3797 /**
3798  * ufshcd_err_handler - handle UFS errors that require s/w attention
3799  * @work: pointer to work structure
3800  */
3801 static void ufshcd_err_handler(struct work_struct *work)
3802 {
3803         struct ufs_hba *hba;
3804         unsigned long flags;
3805         u32 err_xfer = 0;
3806         u32 err_tm = 0;
3807         int err = 0;
3808         int tag;
3809         bool needs_reset = false;
3810
3811         hba = container_of(work, struct ufs_hba, eh_work);
3812
3813         pm_runtime_get_sync(hba->dev);
3814         ufshcd_hold(hba, false);
3815
3816         spin_lock_irqsave(hba->host->host_lock, flags);
3817         if (hba->ufshcd_state == UFSHCD_STATE_RESET)
3818                 goto out;
3819
3820         hba->ufshcd_state = UFSHCD_STATE_RESET;
3821         ufshcd_set_eh_in_progress(hba);
3822
3823         /* Complete requests that have door-bell cleared by h/w */
3824         ufshcd_complete_requests(hba);
3825         if ((hba->saved_err & INT_FATAL_ERRORS) ||
3826             ((hba->saved_err & UIC_ERROR) &&
3827             (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
3828                                    UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
3829                                    UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
3830                 needs_reset = true;
3831
3832         /*
3833          * if host reset is required then skip clearing the pending
3834          * transfers forcefully because they will automatically get
3835          * cleared after link startup.
3836          */
3837         if (needs_reset)
3838                 goto skip_pending_xfer_clear;
3839
3840         /* release lock as clear command might sleep */
3841         spin_unlock_irqrestore(hba->host->host_lock, flags);
3842         /* Clear pending transfer requests */
3843         for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
3844                 if (ufshcd_clear_cmd(hba, tag)) {
3845                         err_xfer = true;
3846                         goto lock_skip_pending_xfer_clear;
3847                 }
3848         }
3849
3850         /* Clear pending task management requests */
3851         for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
3852                 if (ufshcd_clear_tm_cmd(hba, tag)) {
3853                         err_tm = true;
3854                         goto lock_skip_pending_xfer_clear;
3855                 }
3856         }
3857
3858 lock_skip_pending_xfer_clear:
3859         spin_lock_irqsave(hba->host->host_lock, flags);
3860
3861         /* Complete the requests that are cleared by s/w */
3862         ufshcd_complete_requests(hba);
3863
3864         if (err_xfer || err_tm)
3865                 needs_reset = true;
3866
3867 skip_pending_xfer_clear:
3868         /* Fatal errors need reset */
3869         if (needs_reset) {
3870                 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
3871
3872                 /*
3873                  * ufshcd_reset_and_restore() does the link reinitialization
3874                  * which will need atleast one empty doorbell slot to send the
3875                  * device management commands (NOP and query commands).
3876                  * If there is no slot empty at this moment then free up last
3877                  * slot forcefully.
3878                  */
3879                 if (hba->outstanding_reqs == max_doorbells)
3880                         __ufshcd_transfer_req_compl(hba,
3881                                                     (1UL << (hba->nutrs - 1)));
3882
3883                 spin_unlock_irqrestore(hba->host->host_lock, flags);
3884                 err = ufshcd_reset_and_restore(hba);
3885                 spin_lock_irqsave(hba->host->host_lock, flags);
3886                 if (err) {
3887                         dev_err(hba->dev, "%s: reset and restore failed\n",
3888                                         __func__);
3889                         hba->ufshcd_state = UFSHCD_STATE_ERROR;
3890                 }
3891                 /*
3892                  * Inform scsi mid-layer that we did reset and allow to handle
3893                  * Unit Attention properly.
3894                  */
3895                 scsi_report_bus_reset(hba->host, 0);
3896                 hba->saved_err = 0;
3897                 hba->saved_uic_err = 0;
3898         }
3899
3900         if (!needs_reset) {
3901                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
3902                 if (hba->saved_err || hba->saved_uic_err)
3903                         dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
3904                             __func__, hba->saved_err, hba->saved_uic_err);
3905         }
3906
3907         ufshcd_clear_eh_in_progress(hba);
3908
3909 out:
3910         spin_unlock_irqrestore(hba->host->host_lock, flags);
3911         scsi_unblock_requests(hba->host);
3912         ufshcd_release(hba);
3913         pm_runtime_put_sync(hba->dev);
3914 }
3915
3916 /**
3917  * ufshcd_update_uic_error - check and set fatal UIC error flags.
3918  * @hba: per-adapter instance
3919  */
3920 static void ufshcd_update_uic_error(struct ufs_hba *hba)
3921 {
3922         u32 reg;
3923
3924         /* PA_INIT_ERROR is fatal and needs UIC reset */
3925         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
3926         if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
3927                 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
3928
3929         /* UIC NL/TL/DME errors needs software retry */
3930         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
3931         if (reg)
3932                 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
3933
3934         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
3935         if (reg)
3936                 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
3937
3938         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
3939         if (reg)
3940                 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
3941
3942         dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
3943                         __func__, hba->uic_error);
3944 }
3945
3946 /**
3947  * ufshcd_check_errors - Check for errors that need s/w attention
3948  * @hba: per-adapter instance
3949  */
3950 static void ufshcd_check_errors(struct ufs_hba *hba)
3951 {
3952         bool queue_eh_work = false;
3953
3954         if (hba->errors & INT_FATAL_ERRORS)
3955                 queue_eh_work = true;
3956
3957         if (hba->errors & UIC_ERROR) {
3958                 hba->uic_error = 0;
3959                 ufshcd_update_uic_error(hba);
3960                 if (hba->uic_error)
3961                         queue_eh_work = true;
3962         }
3963
3964         if (queue_eh_work) {
3965                 /*
3966                  * update the transfer error masks to sticky bits, let's do this
3967                  * irrespective of current ufshcd_state.
3968                  */
3969                 hba->saved_err |= hba->errors;
3970                 hba->saved_uic_err |= hba->uic_error;
3971
3972                 /* handle fatal errors only when link is functional */
3973                 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
3974                         /* block commands from scsi mid-layer */
3975                         scsi_block_requests(hba->host);
3976
3977                         hba->ufshcd_state = UFSHCD_STATE_ERROR;
3978                         schedule_work(&hba->eh_work);
3979                 }
3980         }
3981         /*
3982          * if (!queue_eh_work) -
3983          * Other errors are either non-fatal where host recovers
3984          * itself without s/w intervention or errors that will be
3985          * handled by the SCSI core layer.
3986          */
3987 }
3988
3989 /**
3990  * ufshcd_tmc_handler - handle task management function completion
3991  * @hba: per adapter instance
3992  */
3993 static void ufshcd_tmc_handler(struct ufs_hba *hba)
3994 {
3995         u32 tm_doorbell;
3996
3997         tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
3998         hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
3999         wake_up(&hba->tm_wq);
4000 }
4001
4002 /**
4003  * ufshcd_sl_intr - Interrupt service routine
4004  * @hba: per adapter instance
4005  * @intr_status: contains interrupts generated by the controller
4006  */
4007 static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
4008 {
4009         hba->errors = UFSHCD_ERROR_MASK & intr_status;
4010         if (hba->errors)
4011                 ufshcd_check_errors(hba);
4012
4013         if (intr_status & UFSHCD_UIC_MASK)
4014                 ufshcd_uic_cmd_compl(hba, intr_status);
4015
4016         if (intr_status & UTP_TASK_REQ_COMPL)
4017                 ufshcd_tmc_handler(hba);
4018
4019         if (intr_status & UTP_TRANSFER_REQ_COMPL)
4020                 ufshcd_transfer_req_compl(hba);
4021 }
4022
4023 /**
4024  * ufshcd_intr - Main interrupt service routine
4025  * @irq: irq number
4026  * @__hba: pointer to adapter instance
4027  *
4028  * Returns IRQ_HANDLED - If interrupt is valid
4029  *              IRQ_NONE - If invalid interrupt
4030  */
4031 static irqreturn_t ufshcd_intr(int irq, void *__hba)
4032 {
4033         u32 intr_status, enabled_intr_status;
4034         irqreturn_t retval = IRQ_NONE;
4035         struct ufs_hba *hba = __hba;
4036
4037         spin_lock(hba->host->host_lock);
4038         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
4039         enabled_intr_status =
4040                 intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
4041
4042         if (intr_status)
4043                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
4044
4045         if (enabled_intr_status) {
4046                 ufshcd_sl_intr(hba, enabled_intr_status);
4047                 retval = IRQ_HANDLED;
4048         }
4049         spin_unlock(hba->host->host_lock);
4050         return retval;
4051 }
4052
4053 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
4054 {
4055         int err = 0;
4056         u32 mask = 1 << tag;
4057         unsigned long flags;
4058
4059         if (!test_bit(tag, &hba->outstanding_tasks))
4060                 goto out;
4061
4062         spin_lock_irqsave(hba->host->host_lock, flags);
4063         ufshcd_writel(hba, ~(1 << tag), REG_UTP_TASK_REQ_LIST_CLEAR);
4064         spin_unlock_irqrestore(hba->host->host_lock, flags);
4065
4066         /* poll for max. 1 sec to clear door bell register by h/w */
4067         err = ufshcd_wait_for_register(hba,
4068                         REG_UTP_TASK_REQ_DOOR_BELL,
4069                         mask, 0, 1000, 1000, true);
4070 out:
4071         return err;
4072 }
4073
4074 /**
4075  * ufshcd_issue_tm_cmd - issues task management commands to controller
4076  * @hba: per adapter instance
4077  * @lun_id: LUN ID to which TM command is sent
4078  * @task_id: task ID to which the TM command is applicable
4079  * @tm_function: task management function opcode
4080  * @tm_response: task management service response return value
4081  *
4082  * Returns non-zero value on error, zero on success.
4083  */
4084 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
4085                 u8 tm_function, u8 *tm_response)
4086 {
4087         struct utp_task_req_desc *task_req_descp;
4088         struct utp_upiu_task_req *task_req_upiup;
4089         struct Scsi_Host *host;
4090         unsigned long flags;
4091         int free_slot;
4092         int err;
4093         int task_tag;
4094
4095         host = hba->host;
4096
4097         /*
4098          * Get free slot, sleep if slots are unavailable.
4099          * Even though we use wait_event() which sleeps indefinitely,
4100          * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
4101          */
4102         wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot));
4103         ufshcd_hold(hba, false);
4104
4105         spin_lock_irqsave(host->host_lock, flags);
4106         task_req_descp = hba->utmrdl_base_addr;
4107         task_req_descp += free_slot;
4108
4109         /* Configure task request descriptor */
4110         task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
4111         task_req_descp->header.dword_2 =
4112                         cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
4113
4114         /* Configure task request UPIU */
4115         task_req_upiup =
4116                 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
4117         task_tag = hba->nutrs + free_slot;
4118         task_req_upiup->header.dword_0 =
4119                 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
4120                                               lun_id, task_tag);
4121         task_req_upiup->header.dword_1 =
4122                 UPIU_HEADER_DWORD(0, tm_function, 0, 0);
4123         /*
4124          * The host shall provide the same value for LUN field in the basic
4125          * header and for Input Parameter.
4126          */
4127         task_req_upiup->input_param1 = cpu_to_be32(lun_id);
4128         task_req_upiup->input_param2 = cpu_to_be32(task_id);
4129
4130         /* send command to the controller */
4131         __set_bit(free_slot, &hba->outstanding_tasks);
4132
4133         /* Make sure descriptors are ready before ringing the task doorbell */
4134         wmb();
4135
4136         ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
4137
4138         spin_unlock_irqrestore(host->host_lock, flags);
4139
4140         /* wait until the task management command is completed */
4141         err = wait_event_timeout(hba->tm_wq,
4142                         test_bit(free_slot, &hba->tm_condition),
4143                         msecs_to_jiffies(TM_CMD_TIMEOUT));
4144         if (!err) {
4145                 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
4146                                 __func__, tm_function);
4147                 if (ufshcd_clear_tm_cmd(hba, free_slot))
4148                         dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
4149                                         __func__, free_slot);
4150                 err = -ETIMEDOUT;
4151         } else {
4152                 err = ufshcd_task_req_compl(hba, free_slot, tm_response);
4153         }
4154
4155         clear_bit(free_slot, &hba->tm_condition);
4156         ufshcd_put_tm_slot(hba, free_slot);
4157         wake_up(&hba->tm_tag_wq);
4158
4159         ufshcd_release(hba);
4160         return err;
4161 }
4162
4163 /**
4164  * ufshcd_eh_device_reset_handler - device reset handler registered to
4165  *                                    scsi layer.
4166  * @cmd: SCSI command pointer
4167  *
4168  * Returns SUCCESS/FAILED
4169  */
4170 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
4171 {
4172         struct Scsi_Host *host;
4173         struct ufs_hba *hba;
4174         unsigned int tag;
4175         u32 pos;
4176         int err;
4177         u8 resp = 0xF;
4178         struct ufshcd_lrb *lrbp;
4179         unsigned long flags;
4180
4181         host = cmd->device->host;
4182         hba = shost_priv(host);
4183         tag = cmd->request->tag;
4184
4185         lrbp = &hba->lrb[tag];
4186         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, 0, UFS_LOGICAL_RESET, &resp);
4187         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4188                 if (!err)
4189                         err = resp;
4190                 goto out;
4191         }
4192
4193         /* clear the commands that were pending for corresponding LUN */
4194         for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
4195                 if (hba->lrb[pos].lun == lrbp->lun) {
4196                         err = ufshcd_clear_cmd(hba, pos);
4197                         if (err)
4198                                 break;
4199                 }
4200         }
4201         spin_lock_irqsave(host->host_lock, flags);
4202         ufshcd_transfer_req_compl(hba);
4203         spin_unlock_irqrestore(host->host_lock, flags);
4204 out:
4205         if (!err) {
4206                 err = SUCCESS;
4207         } else {
4208                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
4209                 err = FAILED;
4210         }
4211         return err;
4212 }
4213
4214 /**
4215  * ufshcd_abort - abort a specific command
4216  * @cmd: SCSI command pointer
4217  *
4218  * Abort the pending command in device by sending UFS_ABORT_TASK task management
4219  * command, and in host controller by clearing the door-bell register. There can
4220  * be race between controller sending the command to the device while abort is
4221  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
4222  * really issued and then try to abort it.
4223  *
4224  * Returns SUCCESS/FAILED
4225  */
4226 static int ufshcd_abort(struct scsi_cmnd *cmd)
4227 {
4228         struct Scsi_Host *host;
4229         struct ufs_hba *hba;
4230         unsigned long flags;
4231         unsigned int tag;
4232         int err = 0;
4233         int poll_cnt;
4234         u8 resp = 0xF;
4235         struct ufshcd_lrb *lrbp;
4236         u32 reg;
4237
4238         host = cmd->device->host;
4239         hba = shost_priv(host);
4240         tag = cmd->request->tag;
4241         if (!ufshcd_valid_tag(hba, tag)) {
4242                 dev_err(hba->dev,
4243                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
4244                         __func__, tag, cmd, cmd->request);
4245                 BUG();
4246         }
4247
4248         ufshcd_hold(hba, false);
4249         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4250         /* If command is already aborted/completed, return SUCCESS */
4251         if (!(test_bit(tag, &hba->outstanding_reqs))) {
4252                 dev_err(hba->dev,
4253                         "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
4254                         __func__, tag, hba->outstanding_reqs, reg);
4255                 goto out;
4256         }
4257
4258         if (!(reg & (1 << tag))) {
4259                 dev_err(hba->dev,
4260                 "%s: cmd was completed, but without a notifying intr, tag = %d",
4261                 __func__, tag);
4262         }
4263
4264         lrbp = &hba->lrb[tag];
4265         for (poll_cnt = 100; poll_cnt; poll_cnt--) {
4266                 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
4267                                 UFS_QUERY_TASK, &resp);
4268                 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
4269                         /* cmd pending in the device */
4270                         break;
4271                 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4272                         /*
4273                          * cmd not pending in the device, check if it is
4274                          * in transition.
4275                          */
4276                         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4277                         if (reg & (1 << tag)) {
4278                                 /* sleep for max. 200us to stabilize */
4279                                 usleep_range(100, 200);
4280                                 continue;
4281                         }
4282                         /* command completed already */
4283                         goto out;
4284                 } else {
4285                         if (!err)
4286                                 err = resp; /* service response error */
4287                         goto out;
4288                 }
4289         }
4290
4291         if (!poll_cnt) {
4292                 err = -EBUSY;
4293                 goto out;
4294         }
4295
4296         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
4297                         UFS_ABORT_TASK, &resp);
4298         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
4299                 if (!err)
4300                         err = resp; /* service response error */
4301                 goto out;
4302         }
4303
4304         err = ufshcd_clear_cmd(hba, tag);
4305         if (err)
4306                 goto out;
4307
4308         scsi_dma_unmap(cmd);
4309
4310         spin_lock_irqsave(host->host_lock, flags);
4311         ufshcd_outstanding_req_clear(hba, tag);
4312         hba->lrb[tag].cmd = NULL;
4313         spin_unlock_irqrestore(host->host_lock, flags);
4314
4315         clear_bit_unlock(tag, &hba->lrb_in_use);
4316         wake_up(&hba->dev_cmd.tag_wq);
4317
4318 out:
4319         if (!err) {
4320                 err = SUCCESS;
4321         } else {
4322                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
4323                 err = FAILED;
4324         }
4325
4326         /*
4327          * This ufshcd_release() corresponds to the original scsi cmd that got
4328          * aborted here (as we won't get any IRQ for it).
4329          */
4330         ufshcd_release(hba);
4331         return err;
4332 }
4333
4334 /**
4335  * ufshcd_host_reset_and_restore - reset and restore host controller
4336  * @hba: per-adapter instance
4337  *
4338  * Note that host controller reset may issue DME_RESET to
4339  * local and remote (device) Uni-Pro stack and the attributes
4340  * are reset to default state.
4341  *
4342  * Returns zero on success, non-zero on failure
4343  */
4344 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
4345 {
4346         int err;
4347         unsigned long flags;
4348
4349         /* Reset the host controller */
4350         spin_lock_irqsave(hba->host->host_lock, flags);
4351         ufshcd_hba_stop(hba, false);
4352         spin_unlock_irqrestore(hba->host->host_lock, flags);
4353
4354         err = ufshcd_hba_enable(hba);
4355         if (err)
4356                 goto out;
4357
4358         /* Establish the link again and restore the device */
4359         err = ufshcd_probe_hba(hba);
4360
4361         if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
4362                 err = -EIO;
4363 out:
4364         if (err)
4365                 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
4366
4367         return err;
4368 }
4369
4370 /**
4371  * ufshcd_reset_and_restore - reset and re-initialize host/device
4372  * @hba: per-adapter instance
4373  *
4374  * Reset and recover device, host and re-establish link. This
4375  * is helpful to recover the communication in fatal error conditions.
4376  *
4377  * Returns zero on success, non-zero on failure
4378  */
4379 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
4380 {
4381         int err = 0;
4382         unsigned long flags;
4383         int retries = MAX_HOST_RESET_RETRIES;
4384
4385         do {
4386                 err = ufshcd_host_reset_and_restore(hba);
4387         } while (err && --retries);
4388
4389         /*
4390          * After reset the door-bell might be cleared, complete
4391          * outstanding requests in s/w here.
4392          */
4393         spin_lock_irqsave(hba->host->host_lock, flags);
4394         ufshcd_transfer_req_compl(hba);
4395         ufshcd_tmc_handler(hba);
4396         spin_unlock_irqrestore(hba->host->host_lock, flags);
4397
4398         return err;
4399 }
4400
4401 /**
4402  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
4403  * @cmd - SCSI command pointer
4404  *
4405  * Returns SUCCESS/FAILED
4406  */
4407 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
4408 {
4409         int err;
4410         unsigned long flags;
4411         struct ufs_hba *hba;
4412
4413         hba = shost_priv(cmd->device->host);
4414
4415         ufshcd_hold(hba, false);
4416         /*
4417          * Check if there is any race with fatal error handling.
4418          * If so, wait for it to complete. Even though fatal error
4419          * handling does reset and restore in some cases, don't assume
4420          * anything out of it. We are just avoiding race here.
4421          */
4422         do {
4423                 spin_lock_irqsave(hba->host->host_lock, flags);
4424                 if (!(work_pending(&hba->eh_work) ||
4425                                 hba->ufshcd_state == UFSHCD_STATE_RESET))
4426                         break;
4427                 spin_unlock_irqrestore(hba->host->host_lock, flags);
4428                 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
4429                 flush_work(&hba->eh_work);
4430         } while (1);
4431
4432         hba->ufshcd_state = UFSHCD_STATE_RESET;
4433         ufshcd_set_eh_in_progress(hba);
4434         spin_unlock_irqrestore(hba->host->host_lock, flags);
4435
4436         err = ufshcd_reset_and_restore(hba);
4437
4438         spin_lock_irqsave(hba->host->host_lock, flags);
4439         if (!err) {
4440                 err = SUCCESS;
4441                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
4442         } else {
4443                 err = FAILED;
4444                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4445         }
4446         ufshcd_clear_eh_in_progress(hba);
4447         spin_unlock_irqrestore(hba->host->host_lock, flags);
4448
4449         ufshcd_release(hba);
4450         return err;
4451 }
4452
4453 /**
4454  * ufshcd_get_max_icc_level - calculate the ICC level
4455  * @sup_curr_uA: max. current supported by the regulator
4456  * @start_scan: row at the desc table to start scan from
4457  * @buff: power descriptor buffer
4458  *
4459  * Returns calculated max ICC level for specific regulator
4460  */
4461 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
4462 {
4463         int i;
4464         int curr_uA;
4465         u16 data;
4466         u16 unit;
4467
4468         for (i = start_scan; i >= 0; i--) {
4469                 data = be16_to_cpu(*((u16 *)(buff + 2*i)));
4470                 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
4471                                                 ATTR_ICC_LVL_UNIT_OFFSET;
4472                 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
4473                 switch (unit) {
4474                 case UFSHCD_NANO_AMP:
4475                         curr_uA = curr_uA / 1000;
4476                         break;
4477                 case UFSHCD_MILI_AMP:
4478                         curr_uA = curr_uA * 1000;
4479                         break;
4480                 case UFSHCD_AMP:
4481                         curr_uA = curr_uA * 1000 * 1000;
4482                         break;
4483                 case UFSHCD_MICRO_AMP:
4484                 default:
4485                         break;
4486                 }
4487                 if (sup_curr_uA >= curr_uA)
4488                         break;
4489         }
4490         if (i < 0) {
4491                 i = 0;
4492                 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
4493         }
4494
4495         return (u32)i;
4496 }
4497
4498 /**
4499  * ufshcd_calc_icc_level - calculate the max ICC level
4500  * In case regulators are not initialized we'll return 0
4501  * @hba: per-adapter instance
4502  * @desc_buf: power descriptor buffer to extract ICC levels from.
4503  * @len: length of desc_buff
4504  *
4505  * Returns calculated ICC level
4506  */
4507 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
4508                                                         u8 *desc_buf, int len)
4509 {
4510         u32 icc_level = 0;
4511
4512         if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
4513                                                 !hba->vreg_info.vccq2) {
4514                 dev_err(hba->dev,
4515                         "%s: Regulator capability was not set, actvIccLevel=%d",
4516                                                         __func__, icc_level);
4517                 goto out;
4518         }
4519
4520         if (hba->vreg_info.vcc)
4521                 icc_level = ufshcd_get_max_icc_level(
4522                                 hba->vreg_info.vcc->max_uA,
4523                                 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
4524                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
4525
4526         if (hba->vreg_info.vccq)
4527                 icc_level = ufshcd_get_max_icc_level(
4528                                 hba->vreg_info.vccq->max_uA,
4529                                 icc_level,
4530                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
4531
4532         if (hba->vreg_info.vccq2)
4533                 icc_level = ufshcd_get_max_icc_level(
4534                                 hba->vreg_info.vccq2->max_uA,
4535                                 icc_level,
4536                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
4537 out:
4538         return icc_level;
4539 }
4540
4541 static void ufshcd_init_icc_levels(struct ufs_hba *hba)
4542 {
4543         int ret;
4544         int buff_len = QUERY_DESC_POWER_MAX_SIZE;
4545         u8 desc_buf[QUERY_DESC_POWER_MAX_SIZE];
4546
4547         ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
4548         if (ret) {
4549                 dev_err(hba->dev,
4550                         "%s: Failed reading power descriptor.len = %d ret = %d",
4551                         __func__, buff_len, ret);
4552                 return;
4553         }
4554
4555         hba->init_prefetch_data.icc_level =
4556                         ufshcd_find_max_sup_active_icc_level(hba,
4557                         desc_buf, buff_len);
4558         dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
4559                         __func__, hba->init_prefetch_data.icc_level);
4560
4561         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
4562                 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
4563                 &hba->init_prefetch_data.icc_level);
4564
4565         if (ret)
4566                 dev_err(hba->dev,
4567                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
4568                         __func__, hba->init_prefetch_data.icc_level , ret);
4569
4570 }
4571
4572 /**
4573  * ufshcd_scsi_add_wlus - Adds required W-LUs
4574  * @hba: per-adapter instance
4575  *
4576  * UFS device specification requires the UFS devices to support 4 well known
4577  * logical units:
4578  *      "REPORT_LUNS" (address: 01h)
4579  *      "UFS Device" (address: 50h)
4580  *      "RPMB" (address: 44h)
4581  *      "BOOT" (address: 30h)
4582  * UFS device's power management needs to be controlled by "POWER CONDITION"
4583  * field of SSU (START STOP UNIT) command. But this "power condition" field
4584  * will take effect only when its sent to "UFS device" well known logical unit
4585  * hence we require the scsi_device instance to represent this logical unit in
4586  * order for the UFS host driver to send the SSU command for power management.
4587
4588  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
4589  * Block) LU so user space process can control this LU. User space may also
4590  * want to have access to BOOT LU.
4591
4592  * This function adds scsi device instances for each of all well known LUs
4593  * (except "REPORT LUNS" LU).
4594  *
4595  * Returns zero on success (all required W-LUs are added successfully),
4596  * non-zero error value on failure (if failed to add any of the required W-LU).
4597  */
4598 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
4599 {
4600         int ret = 0;
4601         struct scsi_device *sdev_rpmb;
4602         struct scsi_device *sdev_boot;
4603
4604         hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
4605                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
4606         if (IS_ERR(hba->sdev_ufs_device)) {
4607                 ret = PTR_ERR(hba->sdev_ufs_device);
4608                 hba->sdev_ufs_device = NULL;
4609                 goto out;
4610         }
4611         scsi_device_put(hba->sdev_ufs_device);
4612
4613         sdev_boot = __scsi_add_device(hba->host, 0, 0,
4614                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
4615         if (IS_ERR(sdev_boot)) {
4616                 ret = PTR_ERR(sdev_boot);
4617                 goto remove_sdev_ufs_device;
4618         }
4619         scsi_device_put(sdev_boot);
4620
4621         sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
4622                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
4623         if (IS_ERR(sdev_rpmb)) {
4624                 ret = PTR_ERR(sdev_rpmb);
4625                 goto remove_sdev_boot;
4626         }
4627         scsi_device_put(sdev_rpmb);
4628         goto out;
4629
4630 remove_sdev_boot:
4631         scsi_remove_device(sdev_boot);
4632 remove_sdev_ufs_device:
4633         scsi_remove_device(hba->sdev_ufs_device);
4634 out:
4635         return ret;
4636 }
4637
4638 static int ufs_get_device_info(struct ufs_hba *hba,
4639                                 struct ufs_device_info *card_data)
4640 {
4641         int err;
4642         u8 model_index;
4643         u8 str_desc_buf[QUERY_DESC_STRING_MAX_SIZE + 1] = {0};
4644         u8 desc_buf[QUERY_DESC_DEVICE_MAX_SIZE];
4645
4646         err = ufshcd_read_device_desc(hba, desc_buf,
4647                                         QUERY_DESC_DEVICE_MAX_SIZE);
4648         if (err) {
4649                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
4650                         __func__, err);
4651                 goto out;
4652         }
4653
4654         /*
4655          * getting vendor (manufacturerID) and Bank Index in big endian
4656          * format
4657          */
4658         card_data->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
4659                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
4660
4661         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
4662
4663         err = ufshcd_read_string_desc(hba, model_index, str_desc_buf,
4664                                         QUERY_DESC_STRING_MAX_SIZE, ASCII_STD);
4665         if (err) {
4666                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
4667                         __func__, err);
4668                 goto out;
4669         }
4670
4671         str_desc_buf[QUERY_DESC_STRING_MAX_SIZE] = '\0';
4672         strlcpy(card_data->model, (str_desc_buf + QUERY_DESC_HDR_SIZE),
4673                 min_t(u8, str_desc_buf[QUERY_DESC_LENGTH_OFFSET],
4674                       MAX_MODEL_LEN));
4675
4676         /* Null terminate the model string */
4677         card_data->model[MAX_MODEL_LEN] = '\0';
4678
4679 out:
4680         return err;
4681 }
4682
4683 void ufs_advertise_fixup_device(struct ufs_hba *hba)
4684 {
4685         int err;
4686         struct ufs_dev_fix *f;
4687         struct ufs_device_info card_data;
4688
4689         card_data.wmanufacturerid = 0;
4690
4691         err = ufs_get_device_info(hba, &card_data);
4692         if (err) {
4693                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
4694                         __func__, err);
4695                 return;
4696         }
4697
4698         for (f = ufs_fixups; f->quirk; f++) {
4699                 if (((f->card.wmanufacturerid == card_data.wmanufacturerid) ||
4700                     (f->card.wmanufacturerid == UFS_ANY_VENDOR)) &&
4701                     (STR_PRFX_EQUAL(f->card.model, card_data.model) ||
4702                      !strcmp(f->card.model, UFS_ANY_MODEL)))
4703                         hba->dev_quirks |= f->quirk;
4704         }
4705 }
4706
4707 /**
4708  * ufshcd_probe_hba - probe hba to detect device and initialize
4709  * @hba: per-adapter instance
4710  *
4711  * Execute link-startup and verify device initialization
4712  */
4713 static int ufshcd_probe_hba(struct ufs_hba *hba)
4714 {
4715         int ret;
4716
4717         ret = ufshcd_link_startup(hba);
4718         if (ret)
4719                 goto out;
4720
4721         ufshcd_init_pwr_info(hba);
4722
4723         /* UniPro link is active now */
4724         ufshcd_set_link_active(hba);
4725
4726         ret = ufshcd_verify_dev_init(hba);
4727         if (ret)
4728                 goto out;
4729
4730         ret = ufshcd_complete_dev_init(hba);
4731         if (ret)
4732                 goto out;
4733
4734         ufs_advertise_fixup_device(hba);
4735
4736         ret = ufshcd_set_vccq_rail_unused(hba,
4737                 (hba->dev_quirks & UFS_DEVICE_NO_VCCQ) ? true : false);
4738         if (ret)
4739                 goto out;
4740
4741         /* UFS device is also active now */
4742         ufshcd_set_ufs_dev_active(hba);
4743         ufshcd_force_reset_auto_bkops(hba);
4744         hba->wlun_dev_clr_ua = true;
4745
4746         if (ufshcd_get_max_pwr_mode(hba)) {
4747                 dev_err(hba->dev,
4748                         "%s: Failed getting max supported power mode\n",
4749                         __func__);
4750         } else {
4751                 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
4752                 if (ret)
4753                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
4754                                         __func__, ret);
4755         }
4756
4757         /* set the state as operational after switching to desired gear */
4758         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
4759         /*
4760          * If we are in error handling context or in power management callbacks
4761          * context, no need to scan the host
4762          */
4763         if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
4764                 bool flag;
4765
4766                 /* clear any previous UFS device information */
4767                 memset(&hba->dev_info, 0, sizeof(hba->dev_info));
4768                 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4769                                 QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
4770                         hba->dev_info.f_power_on_wp_en = flag;
4771
4772                 if (!hba->is_init_prefetch)
4773                         ufshcd_init_icc_levels(hba);
4774
4775                 /* Add required well known logical units to scsi mid layer */
4776                 if (ufshcd_scsi_add_wlus(hba))
4777                         goto out;
4778
4779                 scsi_scan_host(hba->host);
4780                 pm_runtime_put_sync(hba->dev);
4781         }
4782
4783         if (!hba->is_init_prefetch)
4784                 hba->is_init_prefetch = true;
4785
4786         /* Resume devfreq after UFS device is detected */
4787         if (ufshcd_is_clkscaling_enabled(hba))
4788                 devfreq_resume_device(hba->devfreq);
4789
4790 out:
4791         /*
4792          * If we failed to initialize the device or the device is not
4793          * present, turn off the power/clocks etc.
4794          */
4795         if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
4796                 pm_runtime_put_sync(hba->dev);
4797                 ufshcd_hba_exit(hba);
4798         }
4799
4800         return ret;
4801 }
4802
4803 /**
4804  * ufshcd_async_scan - asynchronous execution for probing hba
4805  * @data: data pointer to pass to this function
4806  * @cookie: cookie data
4807  */
4808 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
4809 {
4810         struct ufs_hba *hba = (struct ufs_hba *)data;
4811
4812         ufshcd_probe_hba(hba);
4813 }
4814
4815 static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
4816 {
4817         unsigned long flags;
4818         struct Scsi_Host *host;
4819         struct ufs_hba *hba;
4820         int index;
4821         bool found = false;
4822
4823         if (!scmd || !scmd->device || !scmd->device->host)
4824                 return BLK_EH_NOT_HANDLED;
4825
4826         host = scmd->device->host;
4827         hba = shost_priv(host);
4828         if (!hba)
4829                 return BLK_EH_NOT_HANDLED;
4830
4831         spin_lock_irqsave(host->host_lock, flags);
4832
4833         for_each_set_bit(index, &hba->outstanding_reqs, hba->nutrs) {
4834                 if (hba->lrb[index].cmd == scmd) {
4835                         found = true;
4836                         break;
4837                 }
4838         }
4839
4840         spin_unlock_irqrestore(host->host_lock, flags);
4841
4842         /*
4843          * Bypass SCSI error handling and reset the block layer timer if this
4844          * SCSI command was not actually dispatched to UFS driver, otherwise
4845          * let SCSI layer handle the error as usual.
4846          */
4847         return found ? BLK_EH_NOT_HANDLED : BLK_EH_RESET_TIMER;
4848 }
4849
4850 static struct scsi_host_template ufshcd_driver_template = {
4851         .module                 = THIS_MODULE,
4852         .name                   = UFSHCD,
4853         .proc_name              = UFSHCD,
4854         .queuecommand           = ufshcd_queuecommand,
4855         .slave_alloc            = ufshcd_slave_alloc,
4856         .slave_configure        = ufshcd_slave_configure,
4857         .slave_destroy          = ufshcd_slave_destroy,
4858         .change_queue_depth     = ufshcd_change_queue_depth,
4859         .eh_abort_handler       = ufshcd_abort,
4860         .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
4861         .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
4862         .eh_timed_out           = ufshcd_eh_timed_out,
4863         .this_id                = -1,
4864         .sg_tablesize           = SG_ALL,
4865         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
4866         .can_queue              = UFSHCD_CAN_QUEUE,
4867         .max_host_blocked       = 1,
4868         .track_queue_depth      = 1,
4869 };
4870
4871 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
4872                                    int ua)
4873 {
4874         int ret;
4875
4876         if (!vreg)
4877                 return 0;
4878
4879         ret = regulator_set_load(vreg->reg, ua);
4880         if (ret < 0) {
4881                 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
4882                                 __func__, vreg->name, ua, ret);
4883         }
4884
4885         return ret;
4886 }
4887
4888 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
4889                                          struct ufs_vreg *vreg)
4890 {
4891         if (!vreg)
4892                 return 0;
4893         else if (vreg->unused)
4894                 return 0;
4895         else
4896                 return ufshcd_config_vreg_load(hba->dev, vreg,
4897                                                UFS_VREG_LPM_LOAD_UA);
4898 }
4899
4900 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
4901                                          struct ufs_vreg *vreg)
4902 {
4903         if (!vreg)
4904                 return 0;
4905         else if (vreg->unused)
4906                 return 0;
4907         else
4908                 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
4909 }
4910
4911 static int ufshcd_config_vreg(struct device *dev,
4912                 struct ufs_vreg *vreg, bool on)
4913 {
4914         int ret = 0;
4915         struct regulator *reg = vreg->reg;
4916         const char *name = vreg->name;
4917         int min_uV, uA_load;
4918
4919         BUG_ON(!vreg);
4920
4921         if (regulator_count_voltages(reg) > 0) {
4922                 min_uV = on ? vreg->min_uV : 0;
4923                 ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
4924                 if (ret) {
4925                         dev_err(dev, "%s: %s set voltage failed, err=%d\n",
4926                                         __func__, name, ret);
4927                         goto out;
4928                 }
4929
4930                 uA_load = on ? vreg->max_uA : 0;
4931                 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
4932                 if (ret)
4933                         goto out;
4934         }
4935 out:
4936         return ret;
4937 }
4938
4939 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
4940 {
4941         int ret = 0;
4942
4943         if (!vreg)
4944                 goto out;
4945         else if (vreg->enabled || vreg->unused)
4946                 goto out;
4947
4948         ret = ufshcd_config_vreg(dev, vreg, true);
4949         if (!ret)
4950                 ret = regulator_enable(vreg->reg);
4951
4952         if (!ret)
4953                 vreg->enabled = true;
4954         else
4955                 dev_err(dev, "%s: %s enable failed, err=%d\n",
4956                                 __func__, vreg->name, ret);
4957 out:
4958         return ret;
4959 }
4960
4961 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
4962 {
4963         int ret = 0;
4964
4965         if (!vreg)
4966                 goto out;
4967         else if (!vreg->enabled || vreg->unused)
4968                 goto out;
4969
4970         ret = regulator_disable(vreg->reg);
4971
4972         if (!ret) {
4973                 /* ignore errors on applying disable config */
4974                 ufshcd_config_vreg(dev, vreg, false);
4975                 vreg->enabled = false;
4976         } else {
4977                 dev_err(dev, "%s: %s disable failed, err=%d\n",
4978                                 __func__, vreg->name, ret);
4979         }
4980 out:
4981         return ret;
4982 }
4983
4984 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
4985 {
4986         int ret = 0;
4987         struct device *dev = hba->dev;
4988         struct ufs_vreg_info *info = &hba->vreg_info;
4989
4990         if (!info)
4991                 goto out;
4992
4993         ret = ufshcd_toggle_vreg(dev, info->vcc, on);
4994         if (ret)
4995                 goto out;
4996
4997         ret = ufshcd_toggle_vreg(dev, info->vccq, on);
4998         if (ret)
4999                 goto out;
5000
5001         ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
5002         if (ret)
5003                 goto out;
5004
5005 out:
5006         if (ret) {
5007                 ufshcd_toggle_vreg(dev, info->vccq2, false);
5008                 ufshcd_toggle_vreg(dev, info->vccq, false);
5009                 ufshcd_toggle_vreg(dev, info->vcc, false);
5010         }
5011         return ret;
5012 }
5013
5014 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
5015 {
5016         struct ufs_vreg_info *info = &hba->vreg_info;
5017
5018         if (info)
5019                 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
5020
5021         return 0;
5022 }
5023
5024 static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
5025 {
5026         int ret = 0;
5027
5028         if (!vreg)
5029                 goto out;
5030
5031         vreg->reg = devm_regulator_get(dev, vreg->name);
5032         if (IS_ERR(vreg->reg)) {
5033                 ret = PTR_ERR(vreg->reg);
5034                 dev_err(dev, "%s: %s get failed, err=%d\n",
5035                                 __func__, vreg->name, ret);
5036         }
5037 out:
5038         return ret;
5039 }
5040
5041 static int ufshcd_init_vreg(struct ufs_hba *hba)
5042 {
5043         int ret = 0;
5044         struct device *dev = hba->dev;
5045         struct ufs_vreg_info *info = &hba->vreg_info;
5046
5047         if (!info)
5048                 goto out;
5049
5050         ret = ufshcd_get_vreg(dev, info->vcc);
5051         if (ret)
5052                 goto out;
5053
5054         ret = ufshcd_get_vreg(dev, info->vccq);
5055         if (ret)
5056                 goto out;
5057
5058         ret = ufshcd_get_vreg(dev, info->vccq2);
5059 out:
5060         return ret;
5061 }
5062
5063 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
5064 {
5065         struct ufs_vreg_info *info = &hba->vreg_info;
5066
5067         if (info)
5068                 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
5069
5070         return 0;
5071 }
5072
5073 static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused)
5074 {
5075         int ret = 0;
5076         struct ufs_vreg_info *info = &hba->vreg_info;
5077
5078         if (!info)
5079                 goto out;
5080         else if (!info->vccq)
5081                 goto out;
5082
5083         if (unused) {
5084                 /* shut off the rail here */
5085                 ret = ufshcd_toggle_vreg(hba->dev, info->vccq, false);
5086                 /*
5087                  * Mark this rail as no longer used, so it doesn't get enabled
5088                  * later by mistake
5089                  */
5090                 if (!ret)
5091                         info->vccq->unused = true;
5092         } else {
5093                 /*
5094                  * rail should have been already enabled hence just make sure
5095                  * that unused flag is cleared.
5096                  */
5097                 info->vccq->unused = false;
5098         }
5099 out:
5100         return ret;
5101 }
5102
5103 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
5104                                         bool skip_ref_clk)
5105 {
5106         int ret = 0;
5107         struct ufs_clk_info *clki;
5108         struct list_head *head = &hba->clk_list_head;
5109         unsigned long flags;
5110
5111         if (!head || list_empty(head))
5112                 goto out;
5113
5114         list_for_each_entry(clki, head, list) {
5115                 if (!IS_ERR_OR_NULL(clki->clk)) {
5116                         if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
5117                                 continue;
5118
5119                         if (on && !clki->enabled) {
5120                                 ret = clk_prepare_enable(clki->clk);
5121                                 if (ret) {
5122                                         dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
5123                                                 __func__, clki->name, ret);
5124                                         goto out;
5125                                 }
5126                         } else if (!on && clki->enabled) {
5127                                 clk_disable_unprepare(clki->clk);
5128                         }
5129                         clki->enabled = on;
5130                         dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
5131                                         clki->name, on ? "en" : "dis");
5132                 }
5133         }
5134
5135         ret = ufshcd_vops_setup_clocks(hba, on);
5136 out:
5137         if (ret) {
5138                 list_for_each_entry(clki, head, list) {
5139                         if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
5140                                 clk_disable_unprepare(clki->clk);
5141                 }
5142         } else if (on) {
5143                 spin_lock_irqsave(hba->host->host_lock, flags);
5144                 hba->clk_gating.state = CLKS_ON;
5145                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5146         }
5147         return ret;
5148 }
5149
5150 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
5151 {
5152         return  __ufshcd_setup_clocks(hba, on, false);
5153 }
5154
5155 static int ufshcd_init_clocks(struct ufs_hba *hba)
5156 {
5157         int ret = 0;
5158         struct ufs_clk_info *clki;
5159         struct device *dev = hba->dev;
5160         struct list_head *head = &hba->clk_list_head;
5161
5162         if (!head || list_empty(head))
5163                 goto out;
5164
5165         list_for_each_entry(clki, head, list) {
5166                 if (!clki->name)
5167                         continue;
5168
5169                 clki->clk = devm_clk_get(dev, clki->name);
5170                 if (IS_ERR(clki->clk)) {
5171                         ret = PTR_ERR(clki->clk);
5172                         dev_err(dev, "%s: %s clk get failed, %d\n",
5173                                         __func__, clki->name, ret);
5174                         goto out;
5175                 }
5176
5177                 if (clki->max_freq) {
5178                         ret = clk_set_rate(clki->clk, clki->max_freq);
5179                         if (ret) {
5180                                 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
5181                                         __func__, clki->name,
5182                                         clki->max_freq, ret);
5183                                 goto out;
5184                         }
5185                         clki->curr_freq = clki->max_freq;
5186                 }
5187                 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
5188                                 clki->name, clk_get_rate(clki->clk));
5189         }
5190 out:
5191         return ret;
5192 }
5193
5194 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
5195 {
5196         int err = 0;
5197
5198         if (!hba->vops)
5199                 goto out;
5200
5201         err = ufshcd_vops_init(hba);
5202         if (err)
5203                 goto out;
5204
5205         err = ufshcd_vops_setup_regulators(hba, true);
5206         if (err)
5207                 goto out_exit;
5208
5209         goto out;
5210
5211 out_exit:
5212         ufshcd_vops_exit(hba);
5213 out:
5214         if (err)
5215                 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
5216                         __func__, ufshcd_get_var_name(hba), err);
5217         return err;
5218 }
5219
5220 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
5221 {
5222         if (!hba->vops)
5223                 return;
5224
5225         ufshcd_vops_setup_clocks(hba, false);
5226
5227         ufshcd_vops_setup_regulators(hba, false);
5228
5229         ufshcd_vops_exit(hba);
5230 }
5231
5232 static int ufshcd_hba_init(struct ufs_hba *hba)
5233 {
5234         int err;
5235
5236         /*
5237          * Handle host controller power separately from the UFS device power
5238          * rails as it will help controlling the UFS host controller power
5239          * collapse easily which is different than UFS device power collapse.
5240          * Also, enable the host controller power before we go ahead with rest
5241          * of the initialization here.
5242          */
5243         err = ufshcd_init_hba_vreg(hba);
5244         if (err)
5245                 goto out;
5246
5247         err = ufshcd_setup_hba_vreg(hba, true);
5248         if (err)
5249                 goto out;
5250
5251         err = ufshcd_init_clocks(hba);
5252         if (err)
5253                 goto out_disable_hba_vreg;
5254
5255         err = ufshcd_setup_clocks(hba, true);
5256         if (err)
5257                 goto out_disable_hba_vreg;
5258
5259         err = ufshcd_init_vreg(hba);
5260         if (err)
5261                 goto out_disable_clks;
5262
5263         err = ufshcd_setup_vreg(hba, true);
5264         if (err)
5265                 goto out_disable_clks;
5266
5267         err = ufshcd_variant_hba_init(hba);
5268         if (err)
5269                 goto out_disable_vreg;
5270
5271         hba->is_powered = true;
5272         goto out;
5273
5274 out_disable_vreg:
5275         ufshcd_setup_vreg(hba, false);
5276 out_disable_clks:
5277         ufshcd_setup_clocks(hba, false);
5278 out_disable_hba_vreg:
5279         ufshcd_setup_hba_vreg(hba, false);
5280 out:
5281         return err;
5282 }
5283
5284 static void ufshcd_hba_exit(struct ufs_hba *hba)
5285 {
5286         if (hba->is_powered) {
5287                 ufshcd_variant_hba_exit(hba);
5288                 ufshcd_setup_vreg(hba, false);
5289                 ufshcd_setup_clocks(hba, false);
5290                 ufshcd_setup_hba_vreg(hba, false);
5291                 hba->is_powered = false;
5292         }
5293 }
5294
5295 static int
5296 ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
5297 {
5298         unsigned char cmd[6] = {REQUEST_SENSE,
5299                                 0,
5300                                 0,
5301                                 0,
5302                                 SCSI_SENSE_BUFFERSIZE,
5303                                 0};
5304         char *buffer;
5305         int ret;
5306
5307         buffer = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
5308         if (!buffer) {
5309                 ret = -ENOMEM;
5310                 goto out;
5311         }
5312
5313         ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer,
5314                                 SCSI_SENSE_BUFFERSIZE, NULL,
5315                                 msecs_to_jiffies(1000), 3, NULL, REQ_PM);
5316         if (ret)
5317                 pr_err("%s: failed with err %d\n", __func__, ret);
5318
5319         kfree(buffer);
5320 out:
5321         return ret;
5322 }
5323
5324 /**
5325  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
5326  *                           power mode
5327  * @hba: per adapter instance
5328  * @pwr_mode: device power mode to set
5329  *
5330  * Returns 0 if requested power mode is set successfully
5331  * Returns non-zero if failed to set the requested power mode
5332  */
5333 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
5334                                      enum ufs_dev_pwr_mode pwr_mode)
5335 {
5336         unsigned char cmd[6] = { START_STOP };
5337         struct scsi_sense_hdr sshdr;
5338         struct scsi_device *sdp;
5339         unsigned long flags;
5340         int ret;
5341
5342         spin_lock_irqsave(hba->host->host_lock, flags);
5343         sdp = hba->sdev_ufs_device;
5344         if (sdp) {
5345                 ret = scsi_device_get(sdp);
5346                 if (!ret && !scsi_device_online(sdp)) {
5347                         ret = -ENODEV;
5348                         scsi_device_put(sdp);
5349                 }
5350         } else {
5351                 ret = -ENODEV;
5352         }
5353         spin_unlock_irqrestore(hba->host->host_lock, flags);
5354
5355         if (ret)
5356                 return ret;
5357
5358         /*
5359          * If scsi commands fail, the scsi mid-layer schedules scsi error-
5360          * handling, which would wait for host to be resumed. Since we know
5361          * we are functional while we are here, skip host resume in error
5362          * handling context.
5363          */
5364         hba->host->eh_noresume = 1;
5365         if (hba->wlun_dev_clr_ua) {
5366                 ret = ufshcd_send_request_sense(hba, sdp);
5367                 if (ret)
5368                         goto out;
5369                 /* Unit attention condition is cleared now */
5370                 hba->wlun_dev_clr_ua = false;
5371         }
5372
5373         cmd[4] = pwr_mode << 4;
5374
5375         /*
5376          * Current function would be generally called from the power management
5377          * callbacks hence set the REQ_PM flag so that it doesn't resume the
5378          * already suspended childs.
5379          */
5380         ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr,
5381                                      START_STOP_TIMEOUT, 0, NULL, REQ_PM);
5382         if (ret) {
5383                 sdev_printk(KERN_WARNING, sdp,
5384                             "START_STOP failed for power mode: %d, result %x\n",
5385                             pwr_mode, ret);
5386                 if (driver_byte(ret) & DRIVER_SENSE)
5387                         scsi_print_sense_hdr(sdp, NULL, &sshdr);
5388         }
5389
5390         if (!ret)
5391                 hba->curr_dev_pwr_mode = pwr_mode;
5392 out:
5393         scsi_device_put(sdp);
5394         hba->host->eh_noresume = 0;
5395         return ret;
5396 }
5397
5398 static int ufshcd_link_state_transition(struct ufs_hba *hba,
5399                                         enum uic_link_state req_link_state,
5400                                         int check_for_bkops)
5401 {
5402         int ret = 0;
5403
5404         if (req_link_state == hba->uic_link_state)
5405                 return 0;
5406
5407         if (req_link_state == UIC_LINK_HIBERN8_STATE) {
5408                 ret = ufshcd_uic_hibern8_enter(hba);
5409                 if (!ret)
5410                         ufshcd_set_link_hibern8(hba);
5411                 else
5412                         goto out;
5413         }
5414         /*
5415          * If autobkops is enabled, link can't be turned off because
5416          * turning off the link would also turn off the device.
5417          */
5418         else if ((req_link_state == UIC_LINK_OFF_STATE) &&
5419                    (!check_for_bkops || (check_for_bkops &&
5420                     !hba->auto_bkops_enabled))) {
5421                 /*
5422                  * Change controller state to "reset state" which
5423                  * should also put the link in off/reset state
5424                  */
5425                 ufshcd_hba_stop(hba, true);
5426                 /*
5427                  * TODO: Check if we need any delay to make sure that
5428                  * controller is reset
5429                  */
5430                 ufshcd_set_link_off(hba);
5431         }
5432
5433 out:
5434         return ret;
5435 }
5436
5437 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
5438 {
5439         /*
5440          * If UFS device is either in UFS_Sleep turn off VCC rail to save some
5441          * power.
5442          *
5443          * If UFS device and link is in OFF state, all power supplies (VCC,
5444          * VCCQ, VCCQ2) can be turned off if power on write protect is not
5445          * required. If UFS link is inactive (Hibern8 or OFF state) and device
5446          * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
5447          *
5448          * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
5449          * in low power state which would save some power.
5450          */
5451         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
5452             !hba->dev_info.is_lu_power_on_wp) {
5453                 ufshcd_setup_vreg(hba, false);
5454         } else if (!ufshcd_is_ufs_dev_active(hba)) {
5455                 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
5456                 if (!ufshcd_is_link_active(hba)) {
5457                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
5458                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
5459                 }
5460         }
5461 }
5462
5463 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
5464 {
5465         int ret = 0;
5466
5467         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
5468             !hba->dev_info.is_lu_power_on_wp) {
5469                 ret = ufshcd_setup_vreg(hba, true);
5470         } else if (!ufshcd_is_ufs_dev_active(hba)) {
5471                 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
5472                 if (!ret && !ufshcd_is_link_active(hba)) {
5473                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5474                         if (ret)
5475                                 goto vcc_disable;
5476                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5477                         if (ret)
5478                                 goto vccq_lpm;
5479                 }
5480         }
5481         goto out;
5482
5483 vccq_lpm:
5484         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
5485 vcc_disable:
5486         ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
5487 out:
5488         return ret;
5489 }
5490
5491 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
5492 {
5493         if (ufshcd_is_link_off(hba))
5494                 ufshcd_setup_hba_vreg(hba, false);
5495 }
5496
5497 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
5498 {
5499         if (ufshcd_is_link_off(hba))
5500                 ufshcd_setup_hba_vreg(hba, true);
5501 }
5502
5503 /**
5504  * ufshcd_suspend - helper function for suspend operations
5505  * @hba: per adapter instance
5506  * @pm_op: desired low power operation type
5507  *
5508  * This function will try to put the UFS device and link into low power
5509  * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
5510  * (System PM level).
5511  *
5512  * If this function is called during shutdown, it will make sure that
5513  * both UFS device and UFS link is powered off.
5514  *
5515  * NOTE: UFS device & link must be active before we enter in this function.
5516  *
5517  * Returns 0 for success and non-zero for failure
5518  */
5519 static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
5520 {
5521         int ret = 0;
5522         enum ufs_pm_level pm_lvl;
5523         enum ufs_dev_pwr_mode req_dev_pwr_mode;
5524         enum uic_link_state req_link_state;
5525
5526         hba->pm_op_in_progress = 1;
5527         if (!ufshcd_is_shutdown_pm(pm_op)) {
5528                 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
5529                          hba->rpm_lvl : hba->spm_lvl;
5530                 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
5531                 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
5532         } else {
5533                 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
5534                 req_link_state = UIC_LINK_OFF_STATE;
5535         }
5536
5537         /*
5538          * If we can't transition into any of the low power modes
5539          * just gate the clocks.
5540          */
5541         ufshcd_hold(hba, false);
5542         hba->clk_gating.is_suspended = true;
5543
5544         if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
5545                         req_link_state == UIC_LINK_ACTIVE_STATE) {
5546                 goto disable_clks;
5547         }
5548
5549         if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
5550             (req_link_state == hba->uic_link_state))
5551                 goto out;
5552
5553         /* UFS device & link must be active before we enter in this function */
5554         if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
5555                 ret = -EINVAL;
5556                 goto out;
5557         }
5558
5559         if (ufshcd_is_runtime_pm(pm_op)) {
5560                 if (ufshcd_can_autobkops_during_suspend(hba)) {
5561                         /*
5562                          * The device is idle with no requests in the queue,
5563                          * allow background operations if bkops status shows
5564                          * that performance might be impacted.
5565                          */
5566                         ret = ufshcd_urgent_bkops(hba);
5567                         if (ret)
5568                                 goto enable_gating;
5569                 } else {
5570                         /* make sure that auto bkops is disabled */
5571                         ufshcd_disable_auto_bkops(hba);
5572                 }
5573         }
5574
5575         if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
5576              ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
5577                !ufshcd_is_runtime_pm(pm_op))) {
5578                 /* ensure that bkops is disabled */
5579                 ufshcd_disable_auto_bkops(hba);
5580                 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
5581                 if (ret)
5582                         goto enable_gating;
5583         }
5584
5585         ret = ufshcd_link_state_transition(hba, req_link_state, 1);
5586         if (ret)
5587                 goto set_dev_active;
5588
5589         ufshcd_vreg_set_lpm(hba);
5590
5591 disable_clks:
5592         /*
5593          * The clock scaling needs access to controller registers. Hence, Wait
5594          * for pending clock scaling work to be done before clocks are
5595          * turned off.
5596          */
5597         if (ufshcd_is_clkscaling_enabled(hba)) {
5598                 devfreq_suspend_device(hba->devfreq);
5599                 hba->clk_scaling.window_start_t = 0;
5600         }
5601         /*
5602          * Call vendor specific suspend callback. As these callbacks may access
5603          * vendor specific host controller register space call them before the
5604          * host clocks are ON.
5605          */
5606         ret = ufshcd_vops_suspend(hba, pm_op);
5607         if (ret)
5608                 goto set_link_active;
5609
5610         ret = ufshcd_vops_setup_clocks(hba, false);
5611         if (ret)
5612                 goto vops_resume;
5613
5614         if (!ufshcd_is_link_active(hba))
5615                 ufshcd_setup_clocks(hba, false);
5616         else
5617                 /* If link is active, device ref_clk can't be switched off */
5618                 __ufshcd_setup_clocks(hba, false, true);
5619
5620         hba->clk_gating.state = CLKS_OFF;
5621         /*
5622          * Disable the host irq as host controller as there won't be any
5623          * host controller transaction expected till resume.
5624          */
5625         ufshcd_disable_irq(hba);
5626         /* Put the host controller in low power mode if possible */
5627         ufshcd_hba_vreg_set_lpm(hba);
5628         goto out;
5629
5630 vops_resume:
5631         ufshcd_vops_resume(hba, pm_op);
5632 set_link_active:
5633         ufshcd_vreg_set_hpm(hba);
5634         if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
5635                 ufshcd_set_link_active(hba);
5636         else if (ufshcd_is_link_off(hba))
5637                 ufshcd_host_reset_and_restore(hba);
5638 set_dev_active:
5639         if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
5640                 ufshcd_disable_auto_bkops(hba);
5641 enable_gating:
5642         hba->clk_gating.is_suspended = false;
5643         ufshcd_release(hba);
5644 out:
5645         hba->pm_op_in_progress = 0;
5646         return ret;
5647 }
5648
5649 /**
5650  * ufshcd_resume - helper function for resume operations
5651  * @hba: per adapter instance
5652  * @pm_op: runtime PM or system PM
5653  *
5654  * This function basically brings the UFS device, UniPro link and controller
5655  * to active state.
5656  *
5657  * Returns 0 for success and non-zero for failure
5658  */
5659 static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
5660 {
5661         int ret;
5662         enum uic_link_state old_link_state;
5663
5664         hba->pm_op_in_progress = 1;
5665         old_link_state = hba->uic_link_state;
5666
5667         ufshcd_hba_vreg_set_hpm(hba);
5668         /* Make sure clocks are enabled before accessing controller */
5669         ret = ufshcd_setup_clocks(hba, true);
5670         if (ret)
5671                 goto out;
5672
5673         /* enable the host irq as host controller would be active soon */
5674         ret = ufshcd_enable_irq(hba);
5675         if (ret)
5676                 goto disable_irq_and_vops_clks;
5677
5678         ret = ufshcd_vreg_set_hpm(hba);
5679         if (ret)
5680                 goto disable_irq_and_vops_clks;
5681
5682         /*
5683          * Call vendor specific resume callback. As these callbacks may access
5684          * vendor specific host controller register space call them when the
5685          * host clocks are ON.
5686          */
5687         ret = ufshcd_vops_resume(hba, pm_op);
5688         if (ret)
5689                 goto disable_vreg;
5690
5691         if (ufshcd_is_link_hibern8(hba)) {
5692                 ret = ufshcd_uic_hibern8_exit(hba);
5693                 if (!ret)
5694                         ufshcd_set_link_active(hba);
5695                 else
5696                         goto vendor_suspend;
5697         } else if (ufshcd_is_link_off(hba)) {
5698                 ret = ufshcd_host_reset_and_restore(hba);
5699                 /*
5700                  * ufshcd_host_reset_and_restore() should have already
5701                  * set the link state as active
5702                  */
5703                 if (ret || !ufshcd_is_link_active(hba))
5704                         goto vendor_suspend;
5705         }
5706
5707         if (!ufshcd_is_ufs_dev_active(hba)) {
5708                 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
5709                 if (ret)
5710                         goto set_old_link_state;
5711         }
5712
5713         /*
5714          * If BKOPs operations are urgently needed at this moment then
5715          * keep auto-bkops enabled or else disable it.
5716          */
5717         ufshcd_urgent_bkops(hba);
5718         hba->clk_gating.is_suspended = false;
5719
5720         if (ufshcd_is_clkscaling_enabled(hba))
5721                 devfreq_resume_device(hba->devfreq);
5722
5723         /* Schedule clock gating in case of no access to UFS device yet */
5724         ufshcd_release(hba);
5725         goto out;
5726
5727 set_old_link_state:
5728         ufshcd_link_state_transition(hba, old_link_state, 0);
5729 vendor_suspend:
5730         ufshcd_vops_suspend(hba, pm_op);
5731 disable_vreg:
5732         ufshcd_vreg_set_lpm(hba);
5733 disable_irq_and_vops_clks:
5734         ufshcd_disable_irq(hba);
5735         ufshcd_setup_clocks(hba, false);
5736 out:
5737         hba->pm_op_in_progress = 0;
5738         return ret;
5739 }
5740
5741 /**
5742  * ufshcd_system_suspend - system suspend routine
5743  * @hba: per adapter instance
5744  * @pm_op: runtime PM or system PM
5745  *
5746  * Check the description of ufshcd_suspend() function for more details.
5747  *
5748  * Returns 0 for success and non-zero for failure
5749  */
5750 int ufshcd_system_suspend(struct ufs_hba *hba)
5751 {
5752         int ret = 0;
5753
5754         if (!hba || !hba->is_powered)
5755                 return 0;
5756
5757         if (pm_runtime_suspended(hba->dev)) {
5758                 if (hba->rpm_lvl == hba->spm_lvl)
5759                         /*
5760                          * There is possibility that device may still be in
5761                          * active state during the runtime suspend.
5762                          */
5763                         if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
5764                             hba->curr_dev_pwr_mode) && !hba->auto_bkops_enabled)
5765                                 goto out;
5766
5767                 /*
5768                  * UFS device and/or UFS link low power states during runtime
5769                  * suspend seems to be different than what is expected during
5770                  * system suspend. Hence runtime resume the devic & link and
5771                  * let the system suspend low power states to take effect.
5772                  * TODO: If resume takes longer time, we might have optimize
5773                  * it in future by not resuming everything if possible.
5774                  */
5775                 ret = ufshcd_runtime_resume(hba);
5776                 if (ret)
5777                         goto out;
5778         }
5779
5780         ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
5781 out:
5782         if (!ret)
5783                 hba->is_sys_suspended = true;
5784         return ret;
5785 }
5786 EXPORT_SYMBOL(ufshcd_system_suspend);
5787
5788 /**
5789  * ufshcd_system_resume - system resume routine
5790  * @hba: per adapter instance
5791  *
5792  * Returns 0 for success and non-zero for failure
5793  */
5794
5795 int ufshcd_system_resume(struct ufs_hba *hba)
5796 {
5797         if (!hba || !hba->is_powered || pm_runtime_suspended(hba->dev))
5798                 /*
5799                  * Let the runtime resume take care of resuming
5800                  * if runtime suspended.
5801                  */
5802                 return 0;
5803
5804         return ufshcd_resume(hba, UFS_SYSTEM_PM);
5805 }
5806 EXPORT_SYMBOL(ufshcd_system_resume);
5807
5808 /**
5809  * ufshcd_runtime_suspend - runtime suspend routine
5810  * @hba: per adapter instance
5811  *
5812  * Check the description of ufshcd_suspend() function for more details.
5813  *
5814  * Returns 0 for success and non-zero for failure
5815  */
5816 int ufshcd_runtime_suspend(struct ufs_hba *hba)
5817 {
5818         if (!hba || !hba->is_powered)
5819                 return 0;
5820
5821         return ufshcd_suspend(hba, UFS_RUNTIME_PM);
5822 }
5823 EXPORT_SYMBOL(ufshcd_runtime_suspend);
5824
5825 /**
5826  * ufshcd_runtime_resume - runtime resume routine
5827  * @hba: per adapter instance
5828  *
5829  * This function basically brings the UFS device, UniPro link and controller
5830  * to active state. Following operations are done in this function:
5831  *
5832  * 1. Turn on all the controller related clocks
5833  * 2. Bring the UniPro link out of Hibernate state
5834  * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
5835  *    to active state.
5836  * 4. If auto-bkops is enabled on the device, disable it.
5837  *
5838  * So following would be the possible power state after this function return
5839  * successfully:
5840  *      S1: UFS device in Active state with VCC rail ON
5841  *          UniPro link in Active state
5842  *          All the UFS/UniPro controller clocks are ON
5843  *
5844  * Returns 0 for success and non-zero for failure
5845  */
5846 int ufshcd_runtime_resume(struct ufs_hba *hba)
5847 {
5848         if (!hba || !hba->is_powered)
5849                 return 0;
5850         else
5851                 return ufshcd_resume(hba, UFS_RUNTIME_PM);
5852 }
5853 EXPORT_SYMBOL(ufshcd_runtime_resume);
5854
5855 int ufshcd_runtime_idle(struct ufs_hba *hba)
5856 {
5857         return 0;
5858 }
5859 EXPORT_SYMBOL(ufshcd_runtime_idle);
5860
5861 /**
5862  * ufshcd_shutdown - shutdown routine
5863  * @hba: per adapter instance
5864  *
5865  * This function would power off both UFS device and UFS link.
5866  *
5867  * Returns 0 always to allow force shutdown even in case of errors.
5868  */
5869 int ufshcd_shutdown(struct ufs_hba *hba)
5870 {
5871         int ret = 0;
5872
5873         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
5874                 goto out;
5875
5876         if (pm_runtime_suspended(hba->dev)) {
5877                 ret = ufshcd_runtime_resume(hba);
5878                 if (ret)
5879                         goto out;
5880         }
5881
5882         ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
5883 out:
5884         if (ret)
5885                 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
5886         /* allow force shutdown even in case of errors */
5887         return 0;
5888 }
5889 EXPORT_SYMBOL(ufshcd_shutdown);
5890
5891 /**
5892  * ufshcd_remove - de-allocate SCSI host and host memory space
5893  *              data structure memory
5894  * @hba - per adapter instance
5895  */
5896 void ufshcd_remove(struct ufs_hba *hba)
5897 {
5898         scsi_remove_host(hba->host);
5899         /* disable interrupts */
5900         ufshcd_disable_intr(hba, hba->intr_mask);
5901         ufshcd_hba_stop(hba, true);
5902
5903         scsi_host_put(hba->host);
5904
5905         ufshcd_exit_clk_gating(hba);
5906         if (ufshcd_is_clkscaling_enabled(hba))
5907                 devfreq_remove_device(hba->devfreq);
5908         ufshcd_hba_exit(hba);
5909 }
5910 EXPORT_SYMBOL_GPL(ufshcd_remove);
5911
5912 /**
5913  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
5914  * @hba: pointer to Host Bus Adapter (HBA)
5915  */
5916 void ufshcd_dealloc_host(struct ufs_hba *hba)
5917 {
5918         scsi_host_put(hba->host);
5919 }
5920 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
5921
5922 /**
5923  * ufshcd_set_dma_mask - Set dma mask based on the controller
5924  *                       addressing capability
5925  * @hba: per adapter instance
5926  *
5927  * Returns 0 for success, non-zero for failure
5928  */
5929 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
5930 {
5931         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
5932                 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
5933                         return 0;
5934         }
5935         return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
5936 }
5937
5938 /**
5939  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
5940  * @dev: pointer to device handle
5941  * @hba_handle: driver private handle
5942  * Returns 0 on success, non-zero value on failure
5943  */
5944 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
5945 {
5946         struct Scsi_Host *host;
5947         struct ufs_hba *hba;
5948         int err = 0;
5949
5950         if (!dev) {
5951                 dev_err(dev,
5952                 "Invalid memory reference for dev is NULL\n");
5953                 err = -ENODEV;
5954                 goto out_error;
5955         }
5956
5957         host = scsi_host_alloc(&ufshcd_driver_template,
5958                                 sizeof(struct ufs_hba));
5959         if (!host) {
5960                 dev_err(dev, "scsi_host_alloc failed\n");
5961                 err = -ENOMEM;
5962                 goto out_error;
5963         }
5964         hba = shost_priv(host);
5965         hba->host = host;
5966         hba->dev = dev;
5967         *hba_handle = hba;
5968
5969 out_error:
5970         return err;
5971 }
5972 EXPORT_SYMBOL(ufshcd_alloc_host);
5973
5974 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
5975 {
5976         int ret = 0;
5977         struct ufs_clk_info *clki;
5978         struct list_head *head = &hba->clk_list_head;
5979
5980         if (!head || list_empty(head))
5981                 goto out;
5982
5983         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
5984         if (ret)
5985                 return ret;
5986
5987         list_for_each_entry(clki, head, list) {
5988                 if (!IS_ERR_OR_NULL(clki->clk)) {
5989                         if (scale_up && clki->max_freq) {
5990                                 if (clki->curr_freq == clki->max_freq)
5991                                         continue;
5992                                 ret = clk_set_rate(clki->clk, clki->max_freq);
5993                                 if (ret) {
5994                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
5995                                                 __func__, clki->name,
5996                                                 clki->max_freq, ret);
5997                                         break;
5998                                 }
5999                                 clki->curr_freq = clki->max_freq;
6000
6001                         } else if (!scale_up && clki->min_freq) {
6002                                 if (clki->curr_freq == clki->min_freq)
6003                                         continue;
6004                                 ret = clk_set_rate(clki->clk, clki->min_freq);
6005                                 if (ret) {
6006                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
6007                                                 __func__, clki->name,
6008                                                 clki->min_freq, ret);
6009                                         break;
6010                                 }
6011                                 clki->curr_freq = clki->min_freq;
6012                         }
6013                 }
6014                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
6015                                 clki->name, clk_get_rate(clki->clk));
6016         }
6017
6018         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
6019
6020 out:
6021         return ret;
6022 }
6023
6024 static int ufshcd_devfreq_target(struct device *dev,
6025                                 unsigned long *freq, u32 flags)
6026 {
6027         int err = 0;
6028         struct ufs_hba *hba = dev_get_drvdata(dev);
6029
6030         if (!ufshcd_is_clkscaling_enabled(hba))
6031                 return -EINVAL;
6032
6033         if (*freq == UINT_MAX)
6034                 err = ufshcd_scale_clks(hba, true);
6035         else if (*freq == 0)
6036                 err = ufshcd_scale_clks(hba, false);
6037
6038         return err;
6039 }
6040
6041 static int ufshcd_devfreq_get_dev_status(struct device *dev,
6042                 struct devfreq_dev_status *stat)
6043 {
6044         struct ufs_hba *hba = dev_get_drvdata(dev);
6045         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
6046         unsigned long flags;
6047
6048         if (!ufshcd_is_clkscaling_enabled(hba))
6049                 return -EINVAL;
6050
6051         memset(stat, 0, sizeof(*stat));
6052
6053         spin_lock_irqsave(hba->host->host_lock, flags);
6054         if (!scaling->window_start_t)
6055                 goto start_window;
6056
6057         if (scaling->is_busy_started)
6058                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
6059                                         scaling->busy_start_t));
6060
6061         stat->total_time = jiffies_to_usecs((long)jiffies -
6062                                 (long)scaling->window_start_t);
6063         stat->busy_time = scaling->tot_busy_t;
6064 start_window:
6065         scaling->window_start_t = jiffies;
6066         scaling->tot_busy_t = 0;
6067
6068         if (hba->outstanding_reqs) {
6069                 scaling->busy_start_t = ktime_get();
6070                 scaling->is_busy_started = true;
6071         } else {
6072                 scaling->busy_start_t = ktime_set(0, 0);
6073                 scaling->is_busy_started = false;
6074         }
6075         spin_unlock_irqrestore(hba->host->host_lock, flags);
6076         return 0;
6077 }
6078
6079 static struct devfreq_dev_profile ufs_devfreq_profile = {
6080         .polling_ms     = 100,
6081         .target         = ufshcd_devfreq_target,
6082         .get_dev_status = ufshcd_devfreq_get_dev_status,
6083 };
6084
6085 /**
6086  * ufshcd_init - Driver initialization routine
6087  * @hba: per-adapter instance
6088  * @mmio_base: base register address
6089  * @irq: Interrupt line of device
6090  * Returns 0 on success, non-zero value on failure
6091  */
6092 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
6093 {
6094         int err;
6095         struct Scsi_Host *host = hba->host;
6096         struct device *dev = hba->dev;
6097
6098         if (!mmio_base) {
6099                 dev_err(hba->dev,
6100                 "Invalid memory reference for mmio_base is NULL\n");
6101                 err = -ENODEV;
6102                 goto out_error;
6103         }
6104
6105         hba->mmio_base = mmio_base;
6106         hba->irq = irq;
6107
6108         err = ufshcd_hba_init(hba);
6109         if (err)
6110                 goto out_error;
6111
6112         /* Read capabilities registers */
6113         ufshcd_hba_capabilities(hba);
6114
6115         /* Get UFS version supported by the controller */
6116         hba->ufs_version = ufshcd_get_ufs_version(hba);
6117
6118         /* Get Interrupt bit mask per version */
6119         hba->intr_mask = ufshcd_get_intr_mask(hba);
6120
6121         err = ufshcd_set_dma_mask(hba);
6122         if (err) {
6123                 dev_err(hba->dev, "set dma mask failed\n");
6124                 goto out_disable;
6125         }
6126
6127         /* Allocate memory for host memory space */
6128         err = ufshcd_memory_alloc(hba);
6129         if (err) {
6130                 dev_err(hba->dev, "Memory allocation failed\n");
6131                 goto out_disable;
6132         }
6133
6134         /* Configure LRB */
6135         ufshcd_host_memory_configure(hba);
6136
6137         host->can_queue = hba->nutrs;
6138         host->cmd_per_lun = hba->nutrs;
6139         host->max_id = UFSHCD_MAX_ID;
6140         host->max_lun = UFS_MAX_LUNS;
6141         host->max_channel = UFSHCD_MAX_CHANNEL;
6142         host->unique_id = host->host_no;
6143         host->max_cmd_len = MAX_CDB_SIZE;
6144
6145         hba->max_pwr_info.is_valid = false;
6146
6147         /* Initailize wait queue for task management */
6148         init_waitqueue_head(&hba->tm_wq);
6149         init_waitqueue_head(&hba->tm_tag_wq);
6150
6151         /* Initialize work queues */
6152         INIT_WORK(&hba->eh_work, ufshcd_err_handler);
6153         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
6154
6155         /* Initialize UIC command mutex */
6156         mutex_init(&hba->uic_cmd_mutex);
6157
6158         /* Initialize mutex for device management commands */
6159         mutex_init(&hba->dev_cmd.lock);
6160
6161         /* Initialize device management tag acquire wait queue */
6162         init_waitqueue_head(&hba->dev_cmd.tag_wq);
6163
6164         ufshcd_init_clk_gating(hba);
6165
6166         /*
6167          * In order to avoid any spurious interrupt immediately after
6168          * registering UFS controller interrupt handler, clear any pending UFS
6169          * interrupt status and disable all the UFS interrupts.
6170          */
6171         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
6172                       REG_INTERRUPT_STATUS);
6173         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
6174         /*
6175          * Make sure that UFS interrupts are disabled and any pending interrupt
6176          * status is cleared before registering UFS interrupt handler.
6177          */
6178         mb();
6179
6180         /* IRQ registration */
6181         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
6182         if (err) {
6183                 dev_err(hba->dev, "request irq failed\n");
6184                 goto exit_gating;
6185         } else {
6186                 hba->is_irq_enabled = true;
6187         }
6188
6189         err = scsi_add_host(host, hba->dev);
6190         if (err) {
6191                 dev_err(hba->dev, "scsi_add_host failed\n");
6192                 goto exit_gating;
6193         }
6194
6195         /* Host controller enable */
6196         err = ufshcd_hba_enable(hba);
6197         if (err) {
6198                 dev_err(hba->dev, "Host controller enable failed\n");
6199                 goto out_remove_scsi_host;
6200         }
6201
6202         if (ufshcd_is_clkscaling_enabled(hba)) {
6203                 hba->devfreq = devfreq_add_device(dev, &ufs_devfreq_profile,
6204                                                    "simple_ondemand", NULL);
6205                 if (IS_ERR(hba->devfreq)) {
6206                         dev_err(hba->dev, "Unable to register with devfreq %ld\n",
6207                                         PTR_ERR(hba->devfreq));
6208                         goto out_remove_scsi_host;
6209                 }
6210                 /* Suspend devfreq until the UFS device is detected */
6211                 devfreq_suspend_device(hba->devfreq);
6212                 hba->clk_scaling.window_start_t = 0;
6213         }
6214
6215         /* Hold auto suspend until async scan completes */
6216         pm_runtime_get_sync(dev);
6217
6218         /*
6219          * The device-initialize-sequence hasn't been invoked yet.
6220          * Set the device to power-off state
6221          */
6222         ufshcd_set_ufs_dev_poweroff(hba);
6223
6224         async_schedule(ufshcd_async_scan, hba);
6225
6226         return 0;
6227
6228 out_remove_scsi_host:
6229         scsi_remove_host(hba->host);
6230 exit_gating:
6231         ufshcd_exit_clk_gating(hba);
6232 out_disable:
6233         hba->is_irq_enabled = false;
6234         scsi_host_put(host);
6235         ufshcd_hba_exit(hba);
6236 out_error:
6237         return err;
6238 }
6239 EXPORT_SYMBOL_GPL(ufshcd_init);
6240
6241 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
6242 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
6243 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
6244 MODULE_LICENSE("GPL");
6245 MODULE_VERSION(UFSHCD_DRIVER_VERSION);