drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * This program is free software: you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2  of
  11  * the License as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/delay.h>
  21 #include <linux/slab.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/pm_runtime.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/io.h>
  27 #include <linux/list.h>
  28 #include <linux/dma-mapping.h>
  29
  30 #include <linux/usb/ch9.h>
  31 #include <linux/usb/gadget.h>
  32
  33 #include "debug.h"
  34 #include "core.h"
  35 #include "gadget.h"
  36 #include "io.h"
  37
  38 /**
  39  * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
  40  * @dwc: pointer to our context structure
  41  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  42  *
  43  * Caller should take care of locking. This function will
  44  * return 0 on success or -EINVAL if wrong Test Selector
  45  * is passed
  46  */
  47 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  48 {
  49         u32             reg;
  50
  51         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  52         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  53
  54         switch (mode) {
  55         case TEST_J:
  56         case TEST_K:
  57         case TEST_SE0_NAK:
  58         case TEST_PACKET:
  59         case TEST_FORCE_EN:
  60                 reg |= mode << 1;
  61                 break;
  62         default:
  63                 return -EINVAL;
  64         }
  65
  66         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  67
  68         return 0;
  69 }
  70
  71 /**
  72  * dwc3_gadget_get_link_state - Gets current state of USB Link
  73  * @dwc: pointer to our context structure
  74  *
  75  * Caller should take care of locking. This function will
  76  * return the link state on success (>= 0) or -ETIMEDOUT.
  77  */
  78 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  79 {
  80         u32             reg;
  81
  82         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  83
  84         return DWC3_DSTS_USBLNKST(reg);
  85 }
  86
  87 /**
  88  * dwc3_gadget_set_link_state - Sets USB Link to a particular State
  89  * @dwc: pointer to our context structure
  90  * @state: the state to put link into
  91  *
  92  * Caller should take care of locking. This function will
  93  * return 0 on success or -ETIMEDOUT.
  94  */
  95 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  96 {
  97         int             retries = 10000;
  98         u32             reg;
  99
 100         /*
 101          * Wait until device controller is ready. Only applies to 1.94a and
 102          * later RTL.
 103          */
 104         if (dwc->revision >= DWC3_REVISION_194A) {
 105                 while (--retries) {
 106                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 107                         if (reg & DWC3_DSTS_DCNRD)
 108                                 udelay(5);
 109                         else
 110                                 break;
 111                 }
 112
 113                 if (retries <= 0)
 114                         return -ETIMEDOUT;
 115         }
 116
 117         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 118         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 119
 120         /* set requested state */
 121         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 122         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 123
 124         /*
 125          * The following code is racy when called from dwc3_gadget_wakeup,
 126          * and is not needed, at least on newer versions
 127          */
 128         if (dwc->revision >= DWC3_REVISION_194A)
 129                 return 0;
 130
 131         /* wait for a change in DSTS */
 132         retries = 10000;
 133         while (--retries) {
 134                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 135
 136                 if (DWC3_DSTS_USBLNKST(reg) == state)
 137                         return 0;
 138
 139                 udelay(5);
 140         }
 141
 142         dwc3_trace(trace_dwc3_gadget,
 143                         "link state change request timed out");
 144
 145         return -ETIMEDOUT;
 146 }
 147
 148 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 149 {
 150         dep->trb_enqueue++;
 151         dep->trb_enqueue %= DWC3_TRB_NUM;
 152 }
 153
 154 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 155 {
 156         dep->trb_dequeue++;
 157         dep->trb_dequeue %= DWC3_TRB_NUM;
 158 }
 159
 160 static int dwc3_ep_is_last_trb(unsigned int index)
 161 {
 162         return index == DWC3_TRB_NUM - 1;
 163 }
 164
 165 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 166                 int status)
 167 {
 168         struct dwc3                     *dwc = dep->dwc;
 169         int                             i;
 170
 171         if (req->started) {
 172                 i = 0;
 173                 do {
 174                         dwc3_ep_inc_deq(dep);
 175                         /*
 176                          * Skip LINK TRB. We can't use req->trb and check for
 177                          * DWC3_TRBCTL_LINK_TRB because it points the TRB we
 178                          * just completed (not the LINK TRB).
 179                          */
 180                         if (dwc3_ep_is_last_trb(dep->trb_dequeue))
 181                                 dwc3_ep_inc_deq(dep);
 182                 } while(++i < req->request.num_mapped_sgs);
 183                 req->started = false;
 184         }
 185         list_del(&req->list);
 186         req->trb = NULL;
 187
 188         if (req->request.status == -EINPROGRESS)
 189                 req->request.status = status;
 190
 191         if (dwc->ep0_bounced && dep->number == 0)
 192                 dwc->ep0_bounced = false;
 193         else
 194                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
 195                                 req->direction);
 196
 197         trace_dwc3_gadget_giveback(req);
 198
 199         spin_unlock(&dwc->lock);
 200         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 201         spin_lock(&dwc->lock);
 202 }
 203
 204 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 205 {
 206         u32             timeout = 500;
 207         u32             reg;
 208
 209         trace_dwc3_gadget_generic_cmd(cmd, param);
 210
 211         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 212         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 213
 214         do {
 215                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 216                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 217                         dwc3_trace(trace_dwc3_gadget,
 218                                         "Command Complete --> %d",
 219                                         DWC3_DGCMD_STATUS(reg));
 220                         if (DWC3_DGCMD_STATUS(reg))
 221                                 return -EINVAL;
 222                         return 0;
 223                 }
 224
 225                 /*
 226                  * We can't sleep here, because it's also called from
 227                  * interrupt context.
 228                  */
 229                 timeout--;
 230                 if (!timeout) {
 231                         dwc3_trace(trace_dwc3_gadget,
 232                                         "Command Timed Out");
 233                         return -ETIMEDOUT;
 234                 }
 235                 udelay(1);
 236         } while (1);
 237 }
 238
 239 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 240
 241 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 242                 struct dwc3_gadget_ep_cmd_params *params)
 243 {
 244         struct dwc3             *dwc = dep->dwc;
 245         u32                     timeout = 500;
 246         u32                     reg;
 247
 248         int                     susphy = false;
 249         int                     ret = -EINVAL;
 250
 251         trace_dwc3_gadget_ep_cmd(dep, cmd, params);
 252
 253         /*
 254          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 255          * we're issuing an endpoint command, we must check if
 256          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 257          *
 258          * We will also set SUSPHY bit to what it was before returning as stated
 259          * by the same section on Synopsys databook.
 260          */
 261         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 262                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 263                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 264                         susphy = true;
 265                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 266                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 267                 }
 268         }
 269
 270         if (cmd == DWC3_DEPCMD_STARTTRANSFER) {
 271                 int             needs_wakeup;
 272
 273                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 274                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 275                                 dwc->link_state == DWC3_LINK_STATE_U3);
 276
 277                 if (unlikely(needs_wakeup)) {
 278                         ret = __dwc3_gadget_wakeup(dwc);
 279                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 280                                         ret);
 281                 }
 282         }
 283
 284         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 285         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 286         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 287
 288         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd | DWC3_DEPCMD_CMDACT);
 289         do {
 290                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 291                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 292                         int cmd_status = DWC3_DEPCMD_STATUS(reg);
 293
 294                         dwc3_trace(trace_dwc3_gadget,
 295                                         "Command Complete --> %d",
 296                                         cmd_status);
 297
 298                         switch (cmd_status) {
 299                         case 0:
 300                                 ret = 0;
 301                                 break;
 302                         case DEPEVT_TRANSFER_NO_RESOURCE:
 303                                 dwc3_trace(trace_dwc3_gadget, "%s: no resource available");
 304                                 ret = -EINVAL;
 305                                 break;
 306                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 307                                 /*
 308                                  * SW issues START TRANSFER command to
 309                                  * isochronous ep with future frame interval. If
 310                                  * future interval time has already passed when
 311                                  * core receives the command, it will respond
 312                                  * with an error status of 'Bus Expiry'.
 313                                  *
 314                                  * Instead of always returning -EINVAL, let's
 315                                  * give a hint to the gadget driver that this is
 316                                  * the case by returning -EAGAIN.
 317                                  */
 318                                 dwc3_trace(trace_dwc3_gadget, "%s: bus expiry");
 319                                 ret = -EAGAIN;
 320                                 break;
 321                         default:
 322                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 323                         }
 324
 325                         break;
 326                 }
 327
 328                 /*
 329                  * We can't sleep here, because it is also called from
 330                  * interrupt context.
 331                  */
 332                 timeout--;
 333                 if (!timeout) {
 334                         dwc3_trace(trace_dwc3_gadget,
 335                                         "Command Timed Out");
 336                         ret = -ETIMEDOUT;
 337                         break;
 338                 }
 339         } while (1);
 340
 341         if (unlikely(susphy)) {
 342                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 343                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 344                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 345         }
 346
 347         return ret;
 348 }
 349
 350 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 351 {
 352         struct dwc3 *dwc = dep->dwc;
 353         struct dwc3_gadget_ep_cmd_params params;
 354         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 355
 356         /*
 357          * As of core revision 2.60a the recommended programming model
 358          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 359          * command for IN endpoints. This is to prevent an issue where
 360          * some (non-compliant) hosts may not send ACK TPs for pending
 361          * IN transfers due to a mishandled error condition. Synopsys
 362          * STAR 9000614252.
 363          */
 364         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A))
 365                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 366
 367         memset(&params, 0, sizeof(params));
 368
 369         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 370 }
 371
 372 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 373                 struct dwc3_trb *trb)
 374 {
 375         u32             offset = (char *) trb - (char *) dep->trb_pool;
 376
 377         return dep->trb_pool_dma + offset;
 378 }
 379
 380 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 381 {
 382         struct dwc3             *dwc = dep->dwc;
 383
 384         if (dep->trb_pool)
 385                 return 0;
 386
 387         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 388                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 389                         &dep->trb_pool_dma, GFP_KERNEL);
 390         if (!dep->trb_pool) {
 391                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 392                                 dep->name);
 393                 return -ENOMEM;
 394         }
 395
 396         return 0;
 397 }
 398
 399 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 400 {
 401         struct dwc3             *dwc = dep->dwc;
 402
 403         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 404                         dep->trb_pool, dep->trb_pool_dma);
 405
 406         dep->trb_pool = NULL;
 407         dep->trb_pool_dma = 0;
 408 }
 409
 410 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
 411
 412 /**
 413  * dwc3_gadget_start_config - Configure EP resources
 414  * @dwc: pointer to our controller context structure
 415  * @dep: endpoint that is being enabled
 416  *
 417  * The assignment of transfer resources cannot perfectly follow the
 418  * data book due to the fact that the controller driver does not have
 419  * all knowledge of the configuration in advance. It is given this
 420  * information piecemeal by the composite gadget framework after every
 421  * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
 422  * programming model in this scenario can cause errors. For two
 423  * reasons:
 424  *
 425  * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
 426  * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
 427  * multiple interfaces.
 428  *
 429  * 2) The databook does not mention doing more DEPXFERCFG for new
 430  * endpoint on alt setting (8.1.6).
 431  *
 432  * The following simplified method is used instead:
 433  *
 434  * All hardware endpoints can be assigned a transfer resource and this
 435  * setting will stay persistent until either a core reset or
 436  * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
 437  * do DEPXFERCFG for every hardware endpoint as well. We are
 438  * guaranteed that there are as many transfer resources as endpoints.
 439  *
 440  * This function is called for each endpoint when it is being enabled
 441  * but is triggered only when called for EP0-out, which always happens
 442  * first, and which should only happen in one of the above conditions.
 443  */
 444 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 445 {
 446         struct dwc3_gadget_ep_cmd_params params;
 447         u32                     cmd;
 448         int                     i;
 449         int                     ret;
 450
 451         if (dep->number)
 452                 return 0;
 453
 454         memset(&params, 0x00, sizeof(params));
 455         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 456
 457         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 458         if (ret)
 459                 return ret;
 460
 461         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 462                 struct dwc3_ep *dep = dwc->eps[i];
 463
 464                 if (!dep)
 465                         continue;
 466
 467                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 468                 if (ret)
 469                         return ret;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 476                 const struct usb_endpoint_descriptor *desc,
 477                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 478                 bool ignore, bool restore)
 479 {
 480         struct dwc3_gadget_ep_cmd_params params;
 481
 482         memset(&params, 0x00, sizeof(params));
 483
 484         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 485                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 486
 487         /* Burst size is only needed in SuperSpeed mode */
 488         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 489                 u32 burst = dep->endpoint.maxburst;
 490                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 491         }
 492
 493         if (ignore)
 494                 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
 495
 496         if (restore) {
 497                 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
 498                 params.param2 |= dep->saved_state;
 499         }
 500
 501         params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
 502                 | DWC3_DEPCFG_XFER_NOT_READY_EN;
 503
 504         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 505                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 506                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 507                 dep->stream_capable = true;
 508         }
 509
 510         if (!usb_endpoint_xfer_control(desc))
 511                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 512
 513         /*
 514          * We are doing 1:1 mapping for endpoints, meaning
 515          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 516          * so on. We consider the direction bit as part of the physical
 517          * endpoint number. So USB endpoint 0x81 is 0x03.
 518          */
 519         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 520
 521         /*
 522          * We must use the lower 16 TX FIFOs even though
 523          * HW might have more
 524          */
 525         if (dep->direction)
 526                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 527
 528         if (desc->bInterval) {
 529                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 530                 dep->interval = 1 << (desc->bInterval - 1);
 531         }
 532
 533         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 534 }
 535
 536 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 537 {
 538         struct dwc3_gadget_ep_cmd_params params;
 539
 540         memset(&params, 0x00, sizeof(params));
 541
 542         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 543
 544         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 545                         &params);
 546 }
 547
 548 /**
 549  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 550  * @dep: endpoint to be initialized
 551  * @desc: USB Endpoint Descriptor
 552  *
 553  * Caller should take care of locking
 554  */
 555 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 556                 const struct usb_endpoint_descriptor *desc,
 557                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 558                 bool ignore, bool restore)
 559 {
 560         struct dwc3             *dwc = dep->dwc;
 561         u32                     reg;
 562         int                     ret;
 563
 564         dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);
 565
 566         if (!(dep->flags & DWC3_EP_ENABLED)) {
 567                 ret = dwc3_gadget_start_config(dwc, dep);
 568                 if (ret)
 569                         return ret;
 570         }
 571
 572         ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,
 573                         restore);
 574         if (ret)
 575                 return ret;
 576
 577         if (!(dep->flags & DWC3_EP_ENABLED)) {
 578                 struct dwc3_trb *trb_st_hw;
 579                 struct dwc3_trb *trb_link;
 580
 581                 dep->endpoint.desc = desc;
 582                 dep->comp_desc = comp_desc;
 583                 dep->type = usb_endpoint_type(desc);
 584                 dep->flags |= DWC3_EP_ENABLED;
 585
 586                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 587                 reg |= DWC3_DALEPENA_EP(dep->number);
 588                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 589
 590                 if (usb_endpoint_xfer_control(desc))
 591                         goto out;
 592
 593                 /* Link TRB. The HWO bit is never reset */
 594                 trb_st_hw = &dep->trb_pool[0];
 595
 596                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 597                 memset(trb_link, 0, sizeof(*trb_link));
 598
 599                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 600                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 601                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 602                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 603         }
 604
 605 out:
 606         switch (usb_endpoint_type(desc)) {
 607         case USB_ENDPOINT_XFER_CONTROL:
 608                 /* don't change name */
 609                 break;
 610         case USB_ENDPOINT_XFER_ISOC:
 611                 strlcat(dep->name, "-isoc", sizeof(dep->name));
 612                 break;
 613         case USB_ENDPOINT_XFER_BULK:
 614                 strlcat(dep->name, "-bulk", sizeof(dep->name));
 615                 break;
 616         case USB_ENDPOINT_XFER_INT:
 617                 strlcat(dep->name, "-int", sizeof(dep->name));
 618                 break;
 619         default:
 620                 dev_err(dwc->dev, "invalid endpoint transfer type\n");
 621         }
 622
 623         return 0;
 624 }
 625
 626 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
 627 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 628 {
 629         struct dwc3_request             *req;
 630
 631         if (!list_empty(&dep->started_list)) {
 632                 dwc3_stop_active_transfer(dwc, dep->number, true);
 633
 634                 /* - giveback all requests to gadget driver */
 635                 while (!list_empty(&dep->started_list)) {
 636                         req = next_request(&dep->started_list);
 637
 638                         dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 639                 }
 640         }
 641
 642         while (!list_empty(&dep->pending_list)) {
 643                 req = next_request(&dep->pending_list);
 644
 645                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 646         }
 647 }
 648
 649 /**
 650  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 651  * @dep: the endpoint to disable
 652  *
 653  * This function also removes requests which are currently processed ny the
 654  * hardware and those which are not yet scheduled.
 655  * Caller should take care of locking.
 656  */
 657 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 658 {
 659         struct dwc3             *dwc = dep->dwc;
 660         u32                     reg;
 661
 662         dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);
 663
 664         dwc3_remove_requests(dwc, dep);
 665
 666         /* make sure HW endpoint isn't stalled */
 667         if (dep->flags & DWC3_EP_STALL)
 668                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 669
 670         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 671         reg &= ~DWC3_DALEPENA_EP(dep->number);
 672         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 673
 674         dep->stream_capable = false;
 675         dep->endpoint.desc = NULL;
 676         dep->comp_desc = NULL;
 677         dep->type = 0;
 678         dep->flags = 0;
 679
 680         snprintf(dep->name, sizeof(dep->name), "ep%d%s",
 681                         dep->number >> 1,
 682                         (dep->number & 1) ? "in" : "out");
 683
 684         return 0;
 685 }
 686
 687 /* -------------------------------------------------------------------------- */
 688
 689 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 690                 const struct usb_endpoint_descriptor *desc)
 691 {
 692         return -EINVAL;
 693 }
 694
 695 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 696 {
 697         return -EINVAL;
 698 }
 699
 700 /* -------------------------------------------------------------------------- */
 701
 702 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 703                 const struct usb_endpoint_descriptor *desc)
 704 {
 705         struct dwc3_ep                  *dep;
 706         struct dwc3                     *dwc;
 707         unsigned long                   flags;
 708         int                             ret;
 709
 710         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 711                 pr_debug("dwc3: invalid parameters\n");
 712                 return -EINVAL;
 713         }
 714
 715         if (!desc->wMaxPacketSize) {
 716                 pr_debug("dwc3: missing wMaxPacketSize\n");
 717                 return -EINVAL;
 718         }
 719
 720         dep = to_dwc3_ep(ep);
 721         dwc = dep->dwc;
 722
 723         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 724                                         "%s is already enabled\n",
 725                                         dep->name))
 726                 return 0;
 727
 728         spin_lock_irqsave(&dwc->lock, flags);
 729         ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
 730         spin_unlock_irqrestore(&dwc->lock, flags);
 731
 732         return ret;
 733 }
 734
 735 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 736 {
 737         struct dwc3_ep                  *dep;
 738         struct dwc3                     *dwc;
 739         unsigned long                   flags;
 740         int                             ret;
 741
 742         if (!ep) {
 743                 pr_debug("dwc3: invalid parameters\n");
 744                 return -EINVAL;
 745         }
 746
 747         dep = to_dwc3_ep(ep);
 748         dwc = dep->dwc;
 749
 750         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 751                                         "%s is already disabled\n",
 752                                         dep->name))
 753                 return 0;
 754
 755         spin_lock_irqsave(&dwc->lock, flags);
 756         ret = __dwc3_gadget_ep_disable(dep);
 757         spin_unlock_irqrestore(&dwc->lock, flags);
 758
 759         return ret;
 760 }
 761
 762 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 763         gfp_t gfp_flags)
 764 {
 765         struct dwc3_request             *req;
 766         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 767
 768         req = kzalloc(sizeof(*req), gfp_flags);
 769         if (!req)
 770                 return NULL;
 771
 772         req->epnum      = dep->number;
 773         req->dep        = dep;
 774
 775         trace_dwc3_alloc_request(req);
 776
 777         return &req->request;
 778 }
 779
 780 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 781                 struct usb_request *request)
 782 {
 783         struct dwc3_request             *req = to_dwc3_request(request);
 784
 785         trace_dwc3_free_request(req);
 786         kfree(req);
 787 }
 788
 789 /**
 790  * dwc3_prepare_one_trb - setup one TRB from one request
 791  * @dep: endpoint for which this request is prepared
 792  * @req: dwc3_request pointer
 793  */
 794 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 795                 struct dwc3_request *req, dma_addr_t dma,
 796                 unsigned length, unsigned last, unsigned chain, unsigned node)
 797 {
 798         struct dwc3_trb         *trb;
 799
 800         dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s",
 801                         dep->name, req, (unsigned long long) dma,
 802                         length, last ? " last" : "",
 803                         chain ? " chain" : "");
 804
 805
 806         trb = &dep->trb_pool[dep->trb_enqueue];
 807
 808         if (!req->trb) {
 809                 dwc3_gadget_move_started_request(req);
 810                 req->trb = trb;
 811                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
 812                 req->first_trb_index = dep->trb_enqueue;
 813         }
 814
 815         dwc3_ep_inc_enq(dep);
 816         /* Skip the LINK-TRB */
 817         if (dwc3_ep_is_last_trb(dep->trb_enqueue))
 818                 dwc3_ep_inc_enq(dep);
 819
 820         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 821         trb->bpl = lower_32_bits(dma);
 822         trb->bph = upper_32_bits(dma);
 823
 824         switch (usb_endpoint_type(dep->endpoint.desc)) {
 825         case USB_ENDPOINT_XFER_CONTROL:
 826                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 827                 break;
 828
 829         case USB_ENDPOINT_XFER_ISOC:
 830                 if (!node)
 831                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 832                 else
 833                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 834
 835                 /* always enable Interrupt on Missed ISOC */
 836                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 837                 break;
 838
 839         case USB_ENDPOINT_XFER_BULK:
 840         case USB_ENDPOINT_XFER_INT:
 841                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 842                 break;
 843         default:
 844                 /*
 845                  * This is only possible with faulty memory because we
 846                  * checked it already :)
 847                  */
 848                 BUG();
 849         }
 850
 851         /* always enable Continue on Short Packet */
 852         trb->ctrl |= DWC3_TRB_CTRL_CSP;
 853
 854         if (!req->request.no_interrupt && !chain)
 855                 trb->ctrl |= DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_ISP_IMI;
 856
 857         if (last)
 858                 trb->ctrl |= DWC3_TRB_CTRL_LST;
 859
 860         if (chain)
 861                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
 862
 863         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 864                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
 865
 866         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 867
 868         trace_dwc3_prepare_trb(dep, trb);
 869 }
 870
 871 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 872 {
 873         struct dwc3_trb         *tmp;
 874
 875         /*
 876          * If enqueue & dequeue are equal than it is either full or empty.
 877          *
 878          * One way to know for sure is if the TRB right before us has HWO bit
 879          * set or not. If it has, then we're definitely full and can't fit any
 880          * more transfers in our ring.
 881          */
 882         if (dep->trb_enqueue == dep->trb_dequeue) {
 883                 /* If we're full, enqueue/dequeue are > 0 */
 884                 if (dep->trb_enqueue) {
 885                         tmp = &dep->trb_pool[dep->trb_enqueue - 1];
 886                         if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
 887                                 return 0;
 888                 }
 889
 890                 return DWC3_TRB_NUM - 1;
 891         }
 892
 893         return dep->trb_dequeue - dep->trb_enqueue;
 894 }
 895
 896 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
 897                 struct dwc3_request *req, unsigned int trbs_left)
 898 {
 899         struct usb_request *request = &req->request;
 900         struct scatterlist *sg = request->sg;
 901         struct scatterlist *s;
 902         unsigned int    last = false;
 903         unsigned int    length;
 904         dma_addr_t      dma;
 905         int             i;
 906
 907         for_each_sg(sg, s, request->num_mapped_sgs, i) {
 908                 unsigned chain = true;
 909
 910                 length = sg_dma_len(s);
 911                 dma = sg_dma_address(s);
 912
 913                 if (sg_is_last(s)) {
 914                         if (list_is_last(&req->list, &dep->pending_list))
 915                                 last = true;
 916
 917                         chain = false;
 918                 }
 919
 920                 if (!trbs_left)
 921                         last = true;
 922
 923                 if (last)
 924                         chain = false;
 925
 926                 dwc3_prepare_one_trb(dep, req, dma, length,
 927                                 last, chain, i);
 928
 929                 if (last)
 930                         break;
 931         }
 932 }
 933
 934 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
 935                 struct dwc3_request *req, unsigned int trbs_left)
 936 {
 937         unsigned int    last = false;
 938         unsigned int    length;
 939         dma_addr_t      dma;
 940
 941         dma = req->request.dma;
 942         length = req->request.length;
 943
 944         if (!trbs_left)
 945                 last = true;
 946
 947         /* Is this the last request? */
 948         if (list_is_last(&req->list, &dep->pending_list))
 949                 last = true;
 950
 951         dwc3_prepare_one_trb(dep, req, dma, length,
 952                         last, false, 0);
 953 }
 954
 955 /*
 956  * dwc3_prepare_trbs - setup TRBs from requests
 957  * @dep: endpoint for which requests are being prepared
 958  *
 959  * The function goes through the requests list and sets up TRBs for the
 960  * transfers. The function returns once there are no more TRBs available or
 961  * it runs out of requests.
 962  */
 963 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
 964 {
 965         struct dwc3_request     *req, *n;
 966         u32                     trbs_left;
 967
 968         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 969
 970         trbs_left = dwc3_calc_trbs_left(dep);
 971
 972         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
 973                 if (req->request.num_mapped_sgs > 0)
 974                         dwc3_prepare_one_trb_sg(dep, req, trbs_left--);
 975                 else
 976                         dwc3_prepare_one_trb_linear(dep, req, trbs_left--);
 977
 978                 if (!trbs_left)
 979                         return;
 980         }
 981 }
 982
 983 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param)
 984 {
 985         struct dwc3_gadget_ep_cmd_params params;
 986         struct dwc3_request             *req;
 987         struct dwc3                     *dwc = dep->dwc;
 988         int                             starting;
 989         int                             ret;
 990         u32                             cmd;
 991
 992         starting = !(dep->flags & DWC3_EP_BUSY);
 993
 994         dwc3_prepare_trbs(dep);
 995         req = next_request(&dep->started_list);
 996         if (!req) {
 997                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 998                 return 0;
 999         }
1000
1001         memset(&params, 0, sizeof(params));
1002
1003         if (starting) {
1004                 params.param0 = upper_32_bits(req->trb_dma);
1005                 params.param1 = lower_32_bits(req->trb_dma);
1006                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1007         } else {
1008                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
1009         }
1010
1011         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
1012         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1013         if (ret < 0) {
1014                 /*
1015                  * FIXME we need to iterate over the list of requests
1016                  * here and stop, unmap, free and del each of the linked
1017                  * requests instead of what we do now.
1018                  */
1019                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
1020                                 req->direction);
1021                 list_del(&req->list);
1022                 return ret;
1023         }
1024
1025         dep->flags |= DWC3_EP_BUSY;
1026
1027         if (starting) {
1028                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
1029                 WARN_ON_ONCE(!dep->resource_index);
1030         }
1031
1032         return 0;
1033 }
1034
1035 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1036                 struct dwc3_ep *dep, u32 cur_uf)
1037 {
1038         u32 uf;
1039
1040         if (list_empty(&dep->pending_list)) {
1041                 dwc3_trace(trace_dwc3_gadget,
1042                                 "ISOC ep %s run out for requests",
1043                                 dep->name);
1044                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1045                 return;
1046         }
1047
1048         /* 4 micro frames in the future */
1049         uf = cur_uf + dep->interval * 4;
1050
1051         __dwc3_gadget_kick_transfer(dep, uf);
1052 }
1053
1054 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1055                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1056 {
1057         u32 cur_uf, mask;
1058
1059         mask = ~(dep->interval - 1);
1060         cur_uf = event->parameters & mask;
1061
1062         __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1063 }
1064
1065 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1066 {
1067         struct dwc3             *dwc = dep->dwc;
1068         int                     ret;
1069
1070         if (!dep->endpoint.desc) {
1071                 dwc3_trace(trace_dwc3_gadget,
1072                                 "trying to queue request %p to disabled %s\n",
1073                                 &req->request, dep->endpoint.name);
1074                 return -ESHUTDOWN;
1075         }
1076
1077         if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
1078                                 &req->request, req->dep->name)) {
1079                 dwc3_trace(trace_dwc3_gadget, "request %p belongs to '%s'\n",
1080                                 &req->request, req->dep->name);
1081                 return -EINVAL;
1082         }
1083
1084         req->request.actual     = 0;
1085         req->request.status     = -EINPROGRESS;
1086         req->direction          = dep->direction;
1087         req->epnum              = dep->number;
1088
1089         trace_dwc3_ep_queue(req);
1090
1091         /*
1092          * We only add to our list of requests now and
1093          * start consuming the list once we get XferNotReady
1094          * IRQ.
1095          *
1096          * That way, we avoid doing anything that we don't need
1097          * to do now and defer it until the point we receive a
1098          * particular token from the Host side.
1099          *
1100          * This will also avoid Host cancelling URBs due to too
1101          * many NAKs.
1102          */
1103         ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1104                         dep->direction);
1105         if (ret)
1106                 return ret;
1107
1108         list_add_tail(&req->list, &dep->pending_list);
1109
1110         /*
1111          * If there are no pending requests and the endpoint isn't already
1112          * busy, we will just start the request straight away.
1113          *
1114          * This will save one IRQ (XFER_NOT_READY) and possibly make it a
1115          * little bit faster.
1116          */
1117         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1118                         !usb_endpoint_xfer_int(dep->endpoint.desc) &&
1119                         !(dep->flags & DWC3_EP_BUSY)) {
1120                 ret = __dwc3_gadget_kick_transfer(dep, 0);
1121                 goto out;
1122         }
1123
1124         /*
1125          * There are a few special cases:
1126          *
1127          * 1. XferNotReady with empty list of requests. We need to kick the
1128          *    transfer here in that situation, otherwise we will be NAKing
1129          *    forever. If we get XferNotReady before gadget driver has a
1130          *    chance to queue a request, we will ACK the IRQ but won't be
1131          *    able to receive the data until the next request is queued.
1132          *    The following code is handling exactly that.
1133          *
1134          */
1135         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1136                 /*
1137                  * If xfernotready is already elapsed and it is a case
1138                  * of isoc transfer, then issue END TRANSFER, so that
1139                  * you can receive xfernotready again and can have
1140                  * notion of current microframe.
1141                  */
1142                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1143                         if (list_empty(&dep->started_list)) {
1144                                 dwc3_stop_active_transfer(dwc, dep->number, true);
1145                                 dep->flags = DWC3_EP_ENABLED;
1146                         }
1147                         return 0;
1148                 }
1149
1150                 ret = __dwc3_gadget_kick_transfer(dep, 0);
1151                 if (!ret)
1152                         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1153
1154                 goto out;
1155         }
1156
1157         /*
1158          * 2. XferInProgress on Isoc EP with an active transfer. We need to
1159          *    kick the transfer here after queuing a request, otherwise the
1160          *    core may not see the modified TRB(s).
1161          */
1162         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1163                         (dep->flags & DWC3_EP_BUSY) &&
1164                         !(dep->flags & DWC3_EP_MISSED_ISOC)) {
1165                 WARN_ON_ONCE(!dep->resource_index);
1166                 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index);
1167                 goto out;
1168         }
1169
1170         /*
1171          * 4. Stream Capable Bulk Endpoints. We need to start the transfer
1172          * right away, otherwise host will not know we have streams to be
1173          * handled.
1174          */
1175         if (dep->stream_capable)
1176                 ret = __dwc3_gadget_kick_transfer(dep, 0);
1177
1178 out:
1179         if (ret && ret != -EBUSY)
1180                 dwc3_trace(trace_dwc3_gadget,
1181                                 "%s: failed to kick transfers\n",
1182                                 dep->name);
1183         if (ret == -EBUSY)
1184                 ret = 0;
1185
1186         return ret;
1187 }
1188
1189 static void __dwc3_gadget_ep_zlp_complete(struct usb_ep *ep,
1190                 struct usb_request *request)
1191 {
1192         dwc3_gadget_ep_free_request(ep, request);
1193 }
1194
1195 static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep)
1196 {
1197         struct dwc3_request             *req;
1198         struct usb_request              *request;
1199         struct usb_ep                   *ep = &dep->endpoint;
1200
1201         dwc3_trace(trace_dwc3_gadget, "queueing ZLP\n");
1202         request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1203         if (!request)
1204                 return -ENOMEM;
1205
1206         request->length = 0;
1207         request->buf = dwc->zlp_buf;
1208         request->complete = __dwc3_gadget_ep_zlp_complete;
1209
1210         req = to_dwc3_request(request);
1211
1212         return __dwc3_gadget_ep_queue(dep, req);
1213 }
1214
1215 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1216         gfp_t gfp_flags)
1217 {
1218         struct dwc3_request             *req = to_dwc3_request(request);
1219         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1220         struct dwc3                     *dwc = dep->dwc;
1221
1222         unsigned long                   flags;
1223
1224         int                             ret;
1225
1226         spin_lock_irqsave(&dwc->lock, flags);
1227         ret = __dwc3_gadget_ep_queue(dep, req);
1228
1229         /*
1230          * Okay, here's the thing, if gadget driver has requested for a ZLP by
1231          * setting request->zero, instead of doing magic, we will just queue an
1232          * extra usb_request ourselves so that it gets handled the same way as
1233          * any other request.
1234          */
1235         if (ret == 0 && request->zero && request->length &&
1236             (request->length % ep->maxpacket == 0))
1237                 ret = __dwc3_gadget_ep_queue_zlp(dwc, dep);
1238
1239         spin_unlock_irqrestore(&dwc->lock, flags);
1240
1241         return ret;
1242 }
1243
1244 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1245                 struct usb_request *request)
1246 {
1247         struct dwc3_request             *req = to_dwc3_request(request);
1248         struct dwc3_request             *r = NULL;
1249
1250         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1251         struct dwc3                     *dwc = dep->dwc;
1252
1253         unsigned long                   flags;
1254         int                             ret = 0;
1255
1256         trace_dwc3_ep_dequeue(req);
1257
1258         spin_lock_irqsave(&dwc->lock, flags);
1259
1260         list_for_each_entry(r, &dep->pending_list, list) {
1261                 if (r == req)
1262                         break;
1263         }
1264
1265         if (r != req) {
1266                 list_for_each_entry(r, &dep->started_list, list) {
1267                         if (r == req)
1268                                 break;
1269                 }
1270                 if (r == req) {
1271                         /* wait until it is processed */
1272                         dwc3_stop_active_transfer(dwc, dep->number, true);
1273                         goto out1;
1274                 }
1275                 dev_err(dwc->dev, "request %p was not queued to %s\n",
1276                                 request, ep->name);
1277                 ret = -EINVAL;
1278                 goto out0;
1279         }
1280
1281 out1:
1282         /* giveback the request */
1283         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1284
1285 out0:
1286         spin_unlock_irqrestore(&dwc->lock, flags);
1287
1288         return ret;
1289 }
1290
1291 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1292 {
1293         struct dwc3_gadget_ep_cmd_params        params;
1294         struct dwc3                             *dwc = dep->dwc;
1295         int                                     ret;
1296
1297         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1298                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1299                 return -EINVAL;
1300         }
1301
1302         memset(&params, 0x00, sizeof(params));
1303
1304         if (value) {
1305                 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
1306                                 (!list_empty(&dep->started_list) ||
1307                                  !list_empty(&dep->pending_list)))) {
1308                         dwc3_trace(trace_dwc3_gadget,
1309                                         "%s: pending request, cannot halt",
1310                                         dep->name);
1311                         return -EAGAIN;
1312                 }
1313
1314                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1315                                 &params);
1316                 if (ret)
1317                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1318                                         dep->name);
1319                 else
1320                         dep->flags |= DWC3_EP_STALL;
1321         } else {
1322
1323                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1324                 if (ret)
1325                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1326                                         dep->name);
1327                 else
1328                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1329         }
1330
1331         return ret;
1332 }
1333
1334 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1335 {
1336         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1337         struct dwc3                     *dwc = dep->dwc;
1338
1339         unsigned long                   flags;
1340
1341         int                             ret;
1342
1343         spin_lock_irqsave(&dwc->lock, flags);
1344         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1345         spin_unlock_irqrestore(&dwc->lock, flags);
1346
1347         return ret;
1348 }
1349
1350 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1351 {
1352         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1353         struct dwc3                     *dwc = dep->dwc;
1354         unsigned long                   flags;
1355         int                             ret;
1356
1357         spin_lock_irqsave(&dwc->lock, flags);
1358         dep->flags |= DWC3_EP_WEDGE;
1359
1360         if (dep->number == 0 || dep->number == 1)
1361                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1362         else
1363                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1364         spin_unlock_irqrestore(&dwc->lock, flags);
1365
1366         return ret;
1367 }
1368
1369 /* -------------------------------------------------------------------------- */
1370
1371 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1372         .bLength        = USB_DT_ENDPOINT_SIZE,
1373         .bDescriptorType = USB_DT_ENDPOINT,
1374         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1375 };
1376
1377 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1378         .enable         = dwc3_gadget_ep0_enable,
1379         .disable        = dwc3_gadget_ep0_disable,
1380         .alloc_request  = dwc3_gadget_ep_alloc_request,
1381         .free_request   = dwc3_gadget_ep_free_request,
1382         .queue          = dwc3_gadget_ep0_queue,
1383         .dequeue        = dwc3_gadget_ep_dequeue,
1384         .set_halt       = dwc3_gadget_ep0_set_halt,
1385         .set_wedge      = dwc3_gadget_ep_set_wedge,
1386 };
1387
1388 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1389         .enable         = dwc3_gadget_ep_enable,
1390         .disable        = dwc3_gadget_ep_disable,
1391         .alloc_request  = dwc3_gadget_ep_alloc_request,
1392         .free_request   = dwc3_gadget_ep_free_request,
1393         .queue          = dwc3_gadget_ep_queue,
1394         .dequeue        = dwc3_gadget_ep_dequeue,
1395         .set_halt       = dwc3_gadget_ep_set_halt,
1396         .set_wedge      = dwc3_gadget_ep_set_wedge,
1397 };
1398
1399 /* -------------------------------------------------------------------------- */
1400
1401 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1402 {
1403         struct dwc3             *dwc = gadget_to_dwc(g);
1404         u32                     reg;
1405
1406         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1407         return DWC3_DSTS_SOFFN(reg);
1408 }
1409
1410 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1411 {
1412         unsigned long           timeout;
1413
1414         int                     ret;
1415         u32                     reg;
1416
1417         u8                      link_state;
1418         u8                      speed;
1419
1420         /*
1421          * According to the Databook Remote wakeup request should
1422          * be issued only when the device is in early suspend state.
1423          *
1424          * We can check that via USB Link State bits in DSTS register.
1425          */
1426         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1427
1428         speed = reg & DWC3_DSTS_CONNECTSPD;
1429         if ((speed == DWC3_DSTS_SUPERSPEED) ||
1430             (speed == DWC3_DSTS_SUPERSPEED_PLUS)) {
1431                 dwc3_trace(trace_dwc3_gadget, "no wakeup on SuperSpeed\n");
1432                 return 0;
1433         }
1434
1435         link_state = DWC3_DSTS_USBLNKST(reg);
1436
1437         switch (link_state) {
1438         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1439         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1440                 break;
1441         default:
1442                 dwc3_trace(trace_dwc3_gadget,
1443                                 "can't wakeup from '%s'\n",
1444                                 dwc3_gadget_link_string(link_state));
1445                 return -EINVAL;
1446         }
1447
1448         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1449         if (ret < 0) {
1450                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1451                 return ret;
1452         }
1453
1454         /* Recent versions do this automatically */
1455         if (dwc->revision < DWC3_REVISION_194A) {
1456                 /* write zeroes to Link Change Request */
1457                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1458                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1459                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1460         }
1461
1462         /* poll until Link State changes to ON */
1463         timeout = jiffies + msecs_to_jiffies(100);
1464
1465         while (!time_after(jiffies, timeout)) {
1466                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1467
1468                 /* in HS, means ON */
1469                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1470                         break;
1471         }
1472
1473         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1474                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1475                 return -EINVAL;
1476         }
1477
1478         return 0;
1479 }
1480
1481 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1482 {
1483         struct dwc3             *dwc = gadget_to_dwc(g);
1484         unsigned long           flags;
1485         int                     ret;
1486
1487         spin_lock_irqsave(&dwc->lock, flags);
1488         ret = __dwc3_gadget_wakeup(dwc);
1489         spin_unlock_irqrestore(&dwc->lock, flags);
1490
1491         return ret;
1492 }
1493
1494 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1495                 int is_selfpowered)
1496 {
1497         struct dwc3             *dwc = gadget_to_dwc(g);
1498         unsigned long           flags;
1499
1500         spin_lock_irqsave(&dwc->lock, flags);
1501         g->is_selfpowered = !!is_selfpowered;
1502         spin_unlock_irqrestore(&dwc->lock, flags);
1503
1504         return 0;
1505 }
1506
1507 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1508 {
1509         u32                     reg;
1510         u32                     timeout = 500;
1511
1512         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1513         if (is_on) {
1514                 if (dwc->revision <= DWC3_REVISION_187A) {
1515                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1516                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1517                 }
1518
1519                 if (dwc->revision >= DWC3_REVISION_194A)
1520                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1521                 reg |= DWC3_DCTL_RUN_STOP;
1522
1523                 if (dwc->has_hibernation)
1524                         reg |= DWC3_DCTL_KEEP_CONNECT;
1525
1526                 dwc->pullups_connected = true;
1527         } else {
1528                 reg &= ~DWC3_DCTL_RUN_STOP;
1529
1530                 if (dwc->has_hibernation && !suspend)
1531                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1532
1533                 dwc->pullups_connected = false;
1534         }
1535
1536         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1537
1538         do {
1539                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1540                 if (is_on) {
1541                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1542                                 break;
1543                 } else {
1544                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1545                                 break;
1546                 }
1547                 timeout--;
1548                 if (!timeout)
1549                         return -ETIMEDOUT;
1550                 udelay(1);
1551         } while (1);
1552
1553         dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s",
1554                         dwc->gadget_driver
1555                         ? dwc->gadget_driver->function : "no-function",
1556                         is_on ? "connect" : "disconnect");
1557
1558         return 0;
1559 }
1560
1561 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1562 {
1563         struct dwc3             *dwc = gadget_to_dwc(g);
1564         unsigned long           flags;
1565         int                     ret;
1566
1567         is_on = !!is_on;
1568
1569         spin_lock_irqsave(&dwc->lock, flags);
1570         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1571         spin_unlock_irqrestore(&dwc->lock, flags);
1572
1573         return ret;
1574 }
1575
1576 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1577 {
1578         u32                     reg;
1579
1580         /* Enable all but Start and End of Frame IRQs */
1581         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1582                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1583                         DWC3_DEVTEN_CMDCMPLTEN |
1584                         DWC3_DEVTEN_ERRTICERREN |
1585                         DWC3_DEVTEN_WKUPEVTEN |
1586                         DWC3_DEVTEN_ULSTCNGEN |
1587                         DWC3_DEVTEN_CONNECTDONEEN |
1588                         DWC3_DEVTEN_USBRSTEN |
1589                         DWC3_DEVTEN_DISCONNEVTEN);
1590
1591         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1592 }
1593
1594 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1595 {
1596         /* mask all interrupts */
1597         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1598 }
1599
1600 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1601 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1602
1603 /**
1604  * dwc3_gadget_setup_nump - Calculate and initialize NUMP field of DCFG
1605  * dwc: pointer to our context structure
1606  *
1607  * The following looks like complex but it's actually very simple. In order to
1608  * calculate the number of packets we can burst at once on OUT transfers, we're
1609  * gonna use RxFIFO size.
1610  *
1611  * To calculate RxFIFO size we need two numbers:
1612  * MDWIDTH = size, in bits, of the internal memory bus
1613  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1614  *
1615  * Given these two numbers, the formula is simple:
1616  *
1617  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1618  *
1619  * 24 bytes is for 3x SETUP packets
1620  * 16 bytes is a clock domain crossing tolerance
1621  *
1622  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1623  */
1624 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1625 {
1626         u32 ram2_depth;
1627         u32 mdwidth;
1628         u32 nump;
1629         u32 reg;
1630
1631         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1632         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1633
1634         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1635         nump = min_t(u32, nump, 16);
1636
1637         /* update NumP */
1638         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1639         reg &= ~DWC3_DCFG_NUMP_MASK;
1640         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1641         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1642 }
1643
1644 static int __dwc3_gadget_start(struct dwc3 *dwc)
1645 {
1646         struct dwc3_ep          *dep;
1647         int                     ret = 0;
1648         u32                     reg;
1649
1650         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1651         reg &= ~(DWC3_DCFG_SPEED_MASK);
1652
1653         /**
1654          * WORKAROUND: DWC3 revision < 2.20a have an issue
1655          * which would cause metastability state on Run/Stop
1656          * bit if we try to force the IP to USB2-only mode.
1657          *
1658          * Because of that, we cannot configure the IP to any
1659          * speed other than the SuperSpeed
1660          *
1661          * Refers to:
1662          *
1663          * STAR#9000525659: Clock Domain Crossing on DCTL in
1664          * USB 2.0 Mode
1665          */
1666         if (dwc->revision < DWC3_REVISION_220A) {
1667                 reg |= DWC3_DCFG_SUPERSPEED;
1668         } else {
1669                 switch (dwc->maximum_speed) {
1670                 case USB_SPEED_LOW:
1671                         reg |= DWC3_DSTS_LOWSPEED;
1672                         break;
1673                 case USB_SPEED_FULL:
1674                         reg |= DWC3_DSTS_FULLSPEED1;
1675                         break;
1676                 case USB_SPEED_HIGH:
1677                         reg |= DWC3_DSTS_HIGHSPEED;
1678                         break;
1679                 case USB_SPEED_SUPER_PLUS:
1680                         reg |= DWC3_DSTS_SUPERSPEED_PLUS;
1681                         break;
1682                 default:
1683                         dev_err(dwc->dev, "invalid dwc->maximum_speed (%d)\n",
1684                                 dwc->maximum_speed);
1685                         /* fall through */
1686                 case USB_SPEED_SUPER:
1687                         reg |= DWC3_DCFG_SUPERSPEED;
1688                         break;
1689                 }
1690         }
1691         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1692
1693         /*
1694          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1695          * field instead of letting dwc3 itself calculate that automatically.
1696          *
1697          * This way, we maximize the chances that we'll be able to get several
1698          * bursts of data without going through any sort of endpoint throttling.
1699          */
1700         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1701         reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1702         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1703
1704         dwc3_gadget_setup_nump(dwc);
1705
1706         /* Start with SuperSpeed Default */
1707         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1708
1709         dep = dwc->eps[0];
1710         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1711                         false);
1712         if (ret) {
1713                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1714                 goto err0;
1715         }
1716
1717         dep = dwc->eps[1];
1718         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1719                         false);
1720         if (ret) {
1721                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1722                 goto err1;
1723         }
1724
1725         /* begin to receive SETUP packets */
1726         dwc->ep0state = EP0_SETUP_PHASE;
1727         dwc3_ep0_out_start(dwc);
1728
1729         dwc3_gadget_enable_irq(dwc);
1730
1731         return 0;
1732
1733 err1:
1734         __dwc3_gadget_ep_disable(dwc->eps[0]);
1735
1736 err0:
1737         return ret;
1738 }
1739
1740 static int dwc3_gadget_start(struct usb_gadget *g,
1741                 struct usb_gadget_driver *driver)
1742 {
1743         struct dwc3             *dwc = gadget_to_dwc(g);
1744         unsigned long           flags;
1745         int                     ret = 0;
1746         int                     irq;
1747
1748         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1749         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1750                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1751         if (ret) {
1752                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1753                                 irq, ret);
1754                 goto err0;
1755         }
1756         dwc->irq_gadget = irq;
1757
1758         spin_lock_irqsave(&dwc->lock, flags);
1759         if (dwc->gadget_driver) {
1760                 dev_err(dwc->dev, "%s is already bound to %s\n",
1761                                 dwc->gadget.name,
1762                                 dwc->gadget_driver->driver.name);
1763                 ret = -EBUSY;
1764                 goto err1;
1765         }
1766
1767         dwc->gadget_driver      = driver;
1768
1769         __dwc3_gadget_start(dwc);
1770         spin_unlock_irqrestore(&dwc->lock, flags);
1771
1772         return 0;
1773
1774 err1:
1775         spin_unlock_irqrestore(&dwc->lock, flags);
1776         free_irq(irq, dwc);
1777
1778 err0:
1779         return ret;
1780 }
1781
1782 static void __dwc3_gadget_stop(struct dwc3 *dwc)
1783 {
1784         dwc3_gadget_disable_irq(dwc);
1785         __dwc3_gadget_ep_disable(dwc->eps[0]);
1786         __dwc3_gadget_ep_disable(dwc->eps[1]);
1787 }
1788
1789 static int dwc3_gadget_stop(struct usb_gadget *g)
1790 {
1791         struct dwc3             *dwc = gadget_to_dwc(g);
1792         unsigned long           flags;
1793
1794         spin_lock_irqsave(&dwc->lock, flags);
1795         __dwc3_gadget_stop(dwc);
1796         dwc->gadget_driver      = NULL;
1797         spin_unlock_irqrestore(&dwc->lock, flags);
1798
1799         free_irq(dwc->irq_gadget, dwc->ev_buf);
1800
1801         return 0;
1802 }
1803
1804 static const struct usb_gadget_ops dwc3_gadget_ops = {
1805         .get_frame              = dwc3_gadget_get_frame,
1806         .wakeup                 = dwc3_gadget_wakeup,
1807         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1808         .pullup                 = dwc3_gadget_pullup,
1809         .udc_start              = dwc3_gadget_start,
1810         .udc_stop               = dwc3_gadget_stop,
1811 };
1812
1813 /* -------------------------------------------------------------------------- */
1814
1815 static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1816                 u8 num, u32 direction)
1817 {
1818         struct dwc3_ep                  *dep;
1819         u8                              i;
1820
1821         for (i = 0; i < num; i++) {
1822                 u8 epnum = (i << 1) | (!!direction);
1823
1824                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1825                 if (!dep)
1826                         return -ENOMEM;
1827
1828                 dep->dwc = dwc;
1829                 dep->number = epnum;
1830                 dep->direction = !!direction;
1831                 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
1832                 dwc->eps[epnum] = dep;
1833
1834                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1835                                 (epnum & 1) ? "in" : "out");
1836
1837                 dep->endpoint.name = dep->name;
1838
1839                 dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);
1840
1841                 if (epnum == 0 || epnum == 1) {
1842                         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1843                         dep->endpoint.maxburst = 1;
1844                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1845                         if (!epnum)
1846                                 dwc->gadget.ep0 = &dep->endpoint;
1847                 } else {
1848                         int             ret;
1849
1850                         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
1851                         dep->endpoint.max_streams = 15;
1852                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1853                         list_add_tail(&dep->endpoint.ep_list,
1854                                         &dwc->gadget.ep_list);
1855
1856                         ret = dwc3_alloc_trb_pool(dep);
1857                         if (ret)
1858                                 return ret;
1859                 }
1860
1861                 if (epnum == 0 || epnum == 1) {
1862                         dep->endpoint.caps.type_control = true;
1863                 } else {
1864                         dep->endpoint.caps.type_iso = true;
1865                         dep->endpoint.caps.type_bulk = true;
1866                         dep->endpoint.caps.type_int = true;
1867                 }
1868
1869                 dep->endpoint.caps.dir_in = !!direction;
1870                 dep->endpoint.caps.dir_out = !direction;
1871
1872                 INIT_LIST_HEAD(&dep->pending_list);
1873                 INIT_LIST_HEAD(&dep->started_list);
1874         }
1875
1876         return 0;
1877 }
1878
1879 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1880 {
1881         int                             ret;
1882
1883         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1884
1885         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1886         if (ret < 0) {
1887                 dwc3_trace(trace_dwc3_gadget,
1888                                 "failed to allocate OUT endpoints");
1889                 return ret;
1890         }
1891
1892         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1893         if (ret < 0) {
1894                 dwc3_trace(trace_dwc3_gadget,
1895                                 "failed to allocate IN endpoints");
1896                 return ret;
1897         }
1898
1899         return 0;
1900 }
1901
1902 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1903 {
1904         struct dwc3_ep                  *dep;
1905         u8                              epnum;
1906
1907         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1908                 dep = dwc->eps[epnum];
1909                 if (!dep)
1910                         continue;
1911                 /*
1912                  * Physical endpoints 0 and 1 are special; they form the
1913                  * bi-directional USB endpoint 0.
1914                  *
1915                  * For those two physical endpoints, we don't allocate a TRB
1916                  * pool nor do we add them the endpoints list. Due to that, we
1917                  * shouldn't do these two operations otherwise we would end up
1918                  * with all sorts of bugs when removing dwc3.ko.
1919                  */
1920                 if (epnum != 0 && epnum != 1) {
1921                         dwc3_free_trb_pool(dep);
1922                         list_del(&dep->endpoint.ep_list);
1923                 }
1924
1925                 kfree(dep);
1926         }
1927 }
1928
1929 /* -------------------------------------------------------------------------- */
1930
1931 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1932                 struct dwc3_request *req, struct dwc3_trb *trb,
1933                 const struct dwc3_event_depevt *event, int status)
1934 {
1935         unsigned int            count;
1936         unsigned int            s_pkt = 0;
1937         unsigned int            trb_status;
1938
1939         trace_dwc3_complete_trb(dep, trb);
1940
1941         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1942                 /*
1943                  * We continue despite the error. There is not much we
1944                  * can do. If we don't clean it up we loop forever. If
1945                  * we skip the TRB then it gets overwritten after a
1946                  * while since we use them in a ring buffer. A BUG()
1947                  * would help. Lets hope that if this occurs, someone
1948                  * fixes the root cause instead of looking away :)
1949                  */
1950                 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1951                                 dep->name, trb);
1952         count = trb->size & DWC3_TRB_SIZE_MASK;
1953
1954         if (dep->direction) {
1955                 if (count) {
1956                         trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1957                         if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1958                                 dwc3_trace(trace_dwc3_gadget,
1959                                                 "%s: incomplete IN transfer\n",
1960                                                 dep->name);
1961                                 /*
1962                                  * If missed isoc occurred and there is
1963                                  * no request queued then issue END
1964                                  * TRANSFER, so that core generates
1965                                  * next xfernotready and we will issue
1966                                  * a fresh START TRANSFER.
1967                                  * If there are still queued request
1968                                  * then wait, do not issue either END
1969                                  * or UPDATE TRANSFER, just attach next
1970                                  * request in pending_list during
1971                                  * giveback.If any future queued request
1972                                  * is successfully transferred then we
1973                                  * will issue UPDATE TRANSFER for all
1974                                  * request in the pending_list.
1975                                  */
1976                                 dep->flags |= DWC3_EP_MISSED_ISOC;
1977                         } else {
1978                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1979                                                 dep->name);
1980                                 status = -ECONNRESET;
1981                         }
1982                 } else {
1983                         dep->flags &= ~DWC3_EP_MISSED_ISOC;
1984                 }
1985         } else {
1986                 if (count && (event->status & DEPEVT_STATUS_SHORT))
1987                         s_pkt = 1;
1988         }
1989
1990         /*
1991          * We assume here we will always receive the entire data block
1992          * which we should receive. Meaning, if we program RX to
1993          * receive 4K but we receive only 2K, we assume that's all we
1994          * should receive and we simply bounce the request back to the
1995          * gadget driver for further processing.
1996          */
1997         req->request.actual += req->request.length - count;
1998         if (s_pkt)
1999                 return 1;
2000         if ((event->status & DEPEVT_STATUS_LST) &&
2001                         (trb->ctrl & (DWC3_TRB_CTRL_LST |
2002                                 DWC3_TRB_CTRL_HWO)))
2003                 return 1;
2004         if ((event->status & DEPEVT_STATUS_IOC) &&
2005                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
2006                 return 1;
2007         return 0;
2008 }
2009
2010 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
2011                 const struct dwc3_event_depevt *event, int status)
2012 {
2013         struct dwc3_request     *req;
2014         struct dwc3_trb         *trb;
2015         unsigned int            slot;
2016         unsigned int            i;
2017         int                     ret;
2018
2019         do {
2020                 req = next_request(&dep->started_list);
2021                 if (WARN_ON_ONCE(!req))
2022                         return 1;
2023
2024                 i = 0;
2025                 do {
2026                         slot = req->first_trb_index + i;
2027                         if (slot == DWC3_TRB_NUM - 1)
2028                                 slot++;
2029                         slot %= DWC3_TRB_NUM;
2030                         trb = &dep->trb_pool[slot];
2031
2032                         ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2033                                         event, status);
2034                         if (ret)
2035                                 break;
2036                 } while (++i < req->request.num_mapped_sgs);
2037
2038                 dwc3_gadget_giveback(dep, req, status);
2039
2040                 if (ret)
2041                         break;
2042         } while (1);
2043
2044         /*
2045          * Our endpoint might get disabled by another thread during
2046          * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2047          * early on so DWC3_EP_BUSY flag gets cleared
2048          */
2049         if (!dep->endpoint.desc)
2050                 return 1;
2051
2052         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2053                         list_empty(&dep->started_list)) {
2054                 if (list_empty(&dep->pending_list)) {
2055                         /*
2056                          * If there is no entry in request list then do
2057                          * not issue END TRANSFER now. Just set PENDING
2058                          * flag, so that END TRANSFER is issued when an
2059                          * entry is added into request list.
2060                          */
2061                         dep->flags = DWC3_EP_PENDING_REQUEST;
2062                 } else {
2063                         dwc3_stop_active_transfer(dwc, dep->number, true);
2064                         dep->flags = DWC3_EP_ENABLED;
2065                 }
2066                 return 1;
2067         }
2068
2069         if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
2070                 if ((event->status & DEPEVT_STATUS_IOC) &&
2071                                 (trb->ctrl & DWC3_TRB_CTRL_IOC))
2072                         return 0;
2073         return 1;
2074 }
2075
2076 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
2077                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
2078 {
2079         unsigned                status = 0;
2080         int                     clean_busy;
2081         u32                     is_xfer_complete;
2082
2083         is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
2084
2085         if (event->status & DEPEVT_STATUS_BUSERR)
2086                 status = -ECONNRESET;
2087
2088         clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
2089         if (clean_busy && (!dep->endpoint.desc || is_xfer_complete ||
2090                                 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
2091                 dep->flags &= ~DWC3_EP_BUSY;
2092
2093         /*
2094          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2095          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2096          */
2097         if (dwc->revision < DWC3_REVISION_183A) {
2098                 u32             reg;
2099                 int             i;
2100
2101                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2102                         dep = dwc->eps[i];
2103
2104                         if (!(dep->flags & DWC3_EP_ENABLED))
2105                                 continue;
2106
2107                         if (!list_empty(&dep->started_list))
2108                                 return;
2109                 }
2110
2111                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2112                 reg |= dwc->u1u2;
2113                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2114
2115                 dwc->u1u2 = 0;
2116         }
2117
2118         /*
2119          * Our endpoint might get disabled by another thread during
2120          * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2121          * early on so DWC3_EP_BUSY flag gets cleared
2122          */
2123         if (!dep->endpoint.desc)
2124                 return;
2125
2126         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2127                 int ret;
2128
2129                 ret = __dwc3_gadget_kick_transfer(dep, 0);
2130                 if (!ret || ret == -EBUSY)
2131                         return;
2132         }
2133 }
2134
2135 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2136                 const struct dwc3_event_depevt *event)
2137 {
2138         struct dwc3_ep          *dep;
2139         u8                      epnum = event->endpoint_number;
2140
2141         dep = dwc->eps[epnum];
2142
2143         if (!(dep->flags & DWC3_EP_ENABLED))
2144                 return;
2145
2146         if (epnum == 0 || epnum == 1) {
2147                 dwc3_ep0_interrupt(dwc, event);
2148                 return;
2149         }
2150
2151         switch (event->endpoint_event) {
2152         case DWC3_DEPEVT_XFERCOMPLETE:
2153                 dep->resource_index = 0;
2154
2155                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2156                         dwc3_trace(trace_dwc3_gadget,
2157                                         "%s is an Isochronous endpoint\n",
2158                                         dep->name);
2159                         return;
2160                 }
2161
2162                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2163                 break;
2164         case DWC3_DEPEVT_XFERINPROGRESS:
2165                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2166                 break;
2167         case DWC3_DEPEVT_XFERNOTREADY:
2168                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2169                         dwc3_gadget_start_isoc(dwc, dep, event);
2170                 } else {
2171                         int active;
2172                         int ret;
2173
2174                         active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;
2175
2176                         dwc3_trace(trace_dwc3_gadget, "%s: reason %s",
2177                                         dep->name, active ? "Transfer Active"
2178                                         : "Transfer Not Active");
2179
2180                         ret = __dwc3_gadget_kick_transfer(dep, 0);
2181                         if (!ret || ret == -EBUSY)
2182                                 return;
2183
2184                         dwc3_trace(trace_dwc3_gadget,
2185                                         "%s: failed to kick transfers\n",
2186                                         dep->name);
2187                 }
2188
2189                 break;
2190         case DWC3_DEPEVT_STREAMEVT:
2191                 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
2192                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2193                                         dep->name);
2194                         return;
2195                 }
2196
2197                 switch (event->status) {
2198                 case DEPEVT_STREAMEVT_FOUND:
2199                         dwc3_trace(trace_dwc3_gadget,
2200                                         "Stream %d found and started",
2201                                         event->parameters);
2202
2203                         break;
2204                 case DEPEVT_STREAMEVT_NOTFOUND:
2205                         /* FALLTHROUGH */
2206                 default:
2207                         dwc3_trace(trace_dwc3_gadget,
2208                                         "unable to find suitable stream\n");
2209                 }
2210                 break;
2211         case DWC3_DEPEVT_RXTXFIFOEVT:
2212                 dwc3_trace(trace_dwc3_gadget, "%s FIFO Overrun\n", dep->name);
2213                 break;
2214         case DWC3_DEPEVT_EPCMDCMPLT:
2215                 dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete");
2216                 break;
2217         }
2218 }
2219
2220 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2221 {
2222         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2223                 spin_unlock(&dwc->lock);
2224                 dwc->gadget_driver->disconnect(&dwc->gadget);
2225                 spin_lock(&dwc->lock);
2226         }
2227 }
2228
2229 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2230 {
2231         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2232                 spin_unlock(&dwc->lock);
2233                 dwc->gadget_driver->suspend(&dwc->gadget);
2234                 spin_lock(&dwc->lock);
2235         }
2236 }
2237
2238 static void dwc3_resume_gadget(struct dwc3 *dwc)
2239 {
2240         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2241                 spin_unlock(&dwc->lock);
2242                 dwc->gadget_driver->resume(&dwc->gadget);
2243                 spin_lock(&dwc->lock);
2244         }
2245 }
2246
2247 static void dwc3_reset_gadget(struct dwc3 *dwc)
2248 {
2249         if (!dwc->gadget_driver)
2250                 return;
2251
2252         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2253                 spin_unlock(&dwc->lock);
2254                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2255                 spin_lock(&dwc->lock);
2256         }
2257 }
2258
2259 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2260 {
2261         struct dwc3_ep *dep;
2262         struct dwc3_gadget_ep_cmd_params params;
2263         u32 cmd;
2264         int ret;
2265
2266         dep = dwc->eps[epnum];
2267
2268         if (!dep->resource_index)
2269                 return;
2270
2271         /*
2272          * NOTICE: We are violating what the Databook says about the
2273          * EndTransfer command. Ideally we would _always_ wait for the
2274          * EndTransfer Command Completion IRQ, but that's causing too
2275          * much trouble synchronizing between us and gadget driver.
2276          *
2277          * We have discussed this with the IP Provider and it was
2278          * suggested to giveback all requests here, but give HW some
2279          * extra time to synchronize with the interconnect. We're using
2280          * an arbitrary 100us delay for that.
2281          *
2282          * Note also that a similar handling was tested by Synopsys
2283          * (thanks a lot Paul) and nothing bad has come out of it.
2284          * In short, what we're doing is:
2285          *
2286          * - Issue EndTransfer WITH CMDIOC bit set
2287          * - Wait 100us
2288          */
2289
2290         cmd = DWC3_DEPCMD_ENDTRANSFER;
2291         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2292         cmd |= DWC3_DEPCMD_CMDIOC;
2293         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2294         memset(&params, 0, sizeof(params));
2295         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2296         WARN_ON_ONCE(ret);
2297         dep->resource_index = 0;
2298         dep->flags &= ~DWC3_EP_BUSY;
2299         udelay(100);
2300 }
2301
2302 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2303 {
2304         u32 epnum;
2305
2306         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2307                 struct dwc3_ep *dep;
2308
2309                 dep = dwc->eps[epnum];
2310                 if (!dep)
2311                         continue;
2312
2313                 if (!(dep->flags & DWC3_EP_ENABLED))
2314                         continue;
2315
2316                 dwc3_remove_requests(dwc, dep);
2317         }
2318 }
2319
2320 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2321 {
2322         u32 epnum;
2323
2324         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2325                 struct dwc3_ep *dep;
2326                 int ret;
2327
2328                 dep = dwc->eps[epnum];
2329                 if (!dep)
2330                         continue;
2331
2332                 if (!(dep->flags & DWC3_EP_STALL))
2333                         continue;
2334
2335                 dep->flags &= ~DWC3_EP_STALL;
2336
2337                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2338                 WARN_ON_ONCE(ret);
2339         }
2340 }
2341
2342 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2343 {
2344         int                     reg;
2345
2346         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2347         reg &= ~DWC3_DCTL_INITU1ENA;
2348         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2349
2350         reg &= ~DWC3_DCTL_INITU2ENA;
2351         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2352
2353         dwc3_disconnect_gadget(dwc);
2354
2355         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2356         dwc->setup_packet_pending = false;
2357         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2358 }
2359
2360 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2361 {
2362         u32                     reg;
2363
2364         /*
2365          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2366          * would cause a missing Disconnect Event if there's a
2367          * pending Setup Packet in the FIFO.
2368          *
2369          * There's no suggested workaround on the official Bug
2370          * report, which states that "unless the driver/application
2371          * is doing any special handling of a disconnect event,
2372          * there is no functional issue".
2373          *
2374          * Unfortunately, it turns out that we _do_ some special
2375          * handling of a disconnect event, namely complete all
2376          * pending transfers, notify gadget driver of the
2377          * disconnection, and so on.
2378          *
2379          * Our suggested workaround is to follow the Disconnect
2380          * Event steps here, instead, based on a setup_packet_pending
2381          * flag. Such flag gets set whenever we have a SETUP_PENDING
2382          * status for EP0 TRBs and gets cleared on XferComplete for the
2383          * same endpoint.
2384          *
2385          * Refers to:
2386          *
2387          * STAR#9000466709: RTL: Device : Disconnect event not
2388          * generated if setup packet pending in FIFO
2389          */
2390         if (dwc->revision < DWC3_REVISION_188A) {
2391                 if (dwc->setup_packet_pending)
2392                         dwc3_gadget_disconnect_interrupt(dwc);
2393         }
2394
2395         dwc3_reset_gadget(dwc);
2396
2397         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2398         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2399         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2400         dwc->test_mode = false;
2401
2402         dwc3_stop_active_transfers(dwc);
2403         dwc3_clear_stall_all_ep(dwc);
2404
2405         /* Reset device address to zero */
2406         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2407         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2408         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2409 }
2410
2411 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2412 {
2413         u32 reg;
2414         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2415
2416         /*
2417          * We change the clock only at SS but I dunno why I would want to do
2418          * this. Maybe it becomes part of the power saving plan.
2419          */
2420
2421         if ((speed != DWC3_DSTS_SUPERSPEED) &&
2422             (speed != DWC3_DSTS_SUPERSPEED_PLUS))
2423                 return;
2424
2425         /*
2426          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2427          * each time on Connect Done.
2428          */
2429         if (!usb30_clock)
2430                 return;
2431
2432         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2433         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2434         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2435 }
2436
2437 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2438 {
2439         struct dwc3_ep          *dep;
2440         int                     ret;
2441         u32                     reg;
2442         u8                      speed;
2443
2444         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2445         speed = reg & DWC3_DSTS_CONNECTSPD;
2446         dwc->speed = speed;
2447
2448         dwc3_update_ram_clk_sel(dwc, speed);
2449
2450         switch (speed) {
2451         case DWC3_DCFG_SUPERSPEED_PLUS:
2452                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2453                 dwc->gadget.ep0->maxpacket = 512;
2454                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2455                 break;
2456         case DWC3_DCFG_SUPERSPEED:
2457                 /*
2458                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2459                  * would cause a missing USB3 Reset event.
2460                  *
2461                  * In such situations, we should force a USB3 Reset
2462                  * event by calling our dwc3_gadget_reset_interrupt()
2463                  * routine.
2464                  *
2465                  * Refers to:
2466                  *
2467                  * STAR#9000483510: RTL: SS : USB3 reset event may
2468                  * not be generated always when the link enters poll
2469                  */
2470                 if (dwc->revision < DWC3_REVISION_190A)
2471                         dwc3_gadget_reset_interrupt(dwc);
2472
2473                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2474                 dwc->gadget.ep0->maxpacket = 512;
2475                 dwc->gadget.speed = USB_SPEED_SUPER;
2476                 break;
2477         case DWC3_DCFG_HIGHSPEED:
2478                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2479                 dwc->gadget.ep0->maxpacket = 64;
2480                 dwc->gadget.speed = USB_SPEED_HIGH;
2481                 break;
2482         case DWC3_DCFG_FULLSPEED2:
2483         case DWC3_DCFG_FULLSPEED1:
2484                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2485                 dwc->gadget.ep0->maxpacket = 64;
2486                 dwc->gadget.speed = USB_SPEED_FULL;
2487                 break;
2488         case DWC3_DCFG_LOWSPEED:
2489                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2490                 dwc->gadget.ep0->maxpacket = 8;
2491                 dwc->gadget.speed = USB_SPEED_LOW;
2492                 break;
2493         }
2494
2495         /* Enable USB2 LPM Capability */
2496
2497         if ((dwc->revision > DWC3_REVISION_194A) &&
2498             (speed != DWC3_DCFG_SUPERSPEED) &&
2499             (speed != DWC3_DCFG_SUPERSPEED_PLUS)) {
2500                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2501                 reg |= DWC3_DCFG_LPM_CAP;
2502                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2503
2504                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2505                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2506
2507                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2508
2509                 /*
2510                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2511                  * DCFG.LPMCap is set, core responses with an ACK and the
2512                  * BESL value in the LPM token is less than or equal to LPM
2513                  * NYET threshold.
2514                  */
2515                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2516                                 && dwc->has_lpm_erratum,
2517                                 "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2518
2519                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2520                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2521
2522                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2523         } else {
2524                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2525                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2526                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2527         }
2528
2529         dep = dwc->eps[0];
2530         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2531                         false);
2532         if (ret) {
2533                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2534                 return;
2535         }
2536
2537         dep = dwc->eps[1];
2538         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2539                         false);
2540         if (ret) {
2541                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2542                 return;
2543         }
2544
2545         /*
2546          * Configure PHY via GUSB3PIPECTLn if required.
2547          *
2548          * Update GTXFIFOSIZn
2549          *
2550          * In both cases reset values should be sufficient.
2551          */
2552 }
2553
2554 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2555 {
2556         /*
2557          * TODO take core out of low power mode when that's
2558          * implemented.
2559          */
2560
2561         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2562                 spin_unlock(&dwc->lock);
2563                 dwc->gadget_driver->resume(&dwc->gadget);
2564                 spin_lock(&dwc->lock);
2565         }
2566 }
2567
2568 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2569                 unsigned int evtinfo)
2570 {
2571         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2572         unsigned int            pwropt;
2573
2574         /*
2575          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2576          * Hibernation mode enabled which would show up when device detects
2577          * host-initiated U3 exit.
2578          *
2579          * In that case, device will generate a Link State Change Interrupt
2580          * from U3 to RESUME which is only necessary if Hibernation is
2581          * configured in.
2582          *
2583          * There are no functional changes due to such spurious event and we
2584          * just need to ignore it.
2585          *
2586          * Refers to:
2587          *
2588          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2589          * operational mode
2590          */
2591         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2592         if ((dwc->revision < DWC3_REVISION_250A) &&
2593                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2594                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2595                                 (next == DWC3_LINK_STATE_RESUME)) {
2596                         dwc3_trace(trace_dwc3_gadget,
2597                                         "ignoring transition U3 -> Resume");
2598                         return;
2599                 }
2600         }
2601
2602         /*
2603          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2604          * on the link partner, the USB session might do multiple entry/exit
2605          * of low power states before a transfer takes place.
2606          *
2607          * Due to this problem, we might experience lower throughput. The
2608          * suggested workaround is to disable DCTL[12:9] bits if we're
2609          * transitioning from U1/U2 to U0 and enable those bits again
2610          * after a transfer completes and there are no pending transfers
2611          * on any of the enabled endpoints.
2612          *
2613          * This is the first half of that workaround.
2614          *
2615          * Refers to:
2616          *
2617          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2618          * core send LGO_Ux entering U0
2619          */
2620         if (dwc->revision < DWC3_REVISION_183A) {
2621                 if (next == DWC3_LINK_STATE_U0) {
2622                         u32     u1u2;
2623                         u32     reg;
2624
2625                         switch (dwc->link_state) {
2626                         case DWC3_LINK_STATE_U1:
2627                         case DWC3_LINK_STATE_U2:
2628                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2629                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2630                                                 | DWC3_DCTL_ACCEPTU2ENA
2631                                                 | DWC3_DCTL_INITU1ENA
2632                                                 | DWC3_DCTL_ACCEPTU1ENA);
2633
2634                                 if (!dwc->u1u2)
2635                                         dwc->u1u2 = reg & u1u2;
2636
2637                                 reg &= ~u1u2;
2638
2639                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2640                                 break;
2641                         default:
2642                                 /* do nothing */
2643                                 break;
2644                         }
2645                 }
2646         }
2647
2648         switch (next) {
2649         case DWC3_LINK_STATE_U1:
2650                 if (dwc->speed == USB_SPEED_SUPER)
2651                         dwc3_suspend_gadget(dwc);
2652                 break;
2653         case DWC3_LINK_STATE_U2:
2654         case DWC3_LINK_STATE_U3:
2655                 dwc3_suspend_gadget(dwc);
2656                 break;
2657         case DWC3_LINK_STATE_RESUME:
2658                 dwc3_resume_gadget(dwc);
2659                 break;
2660         default:
2661                 /* do nothing */
2662                 break;
2663         }
2664
2665         dwc->link_state = next;
2666 }
2667
2668 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2669                 unsigned int evtinfo)
2670 {
2671         unsigned int is_ss = evtinfo & BIT(4);
2672
2673         /**
2674          * WORKAROUND: DWC3 revison 2.20a with hibernation support
2675          * have a known issue which can cause USB CV TD.9.23 to fail
2676          * randomly.
2677          *
2678          * Because of this issue, core could generate bogus hibernation
2679          * events which SW needs to ignore.
2680          *
2681          * Refers to:
2682          *
2683          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2684          * Device Fallback from SuperSpeed
2685          */
2686         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2687                 return;
2688
2689         /* enter hibernation here */
2690 }
2691
2692 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2693                 const struct dwc3_event_devt *event)
2694 {
2695         switch (event->type) {
2696         case DWC3_DEVICE_EVENT_DISCONNECT:
2697                 dwc3_gadget_disconnect_interrupt(dwc);
2698                 break;
2699         case DWC3_DEVICE_EVENT_RESET:
2700                 dwc3_gadget_reset_interrupt(dwc);
2701                 break;
2702         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2703                 dwc3_gadget_conndone_interrupt(dwc);
2704                 break;
2705         case DWC3_DEVICE_EVENT_WAKEUP:
2706                 dwc3_gadget_wakeup_interrupt(dwc);
2707                 break;
2708         case DWC3_DEVICE_EVENT_HIBER_REQ:
2709                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2710                                         "unexpected hibernation event\n"))
2711                         break;
2712
2713                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2714                 break;
2715         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2716                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2717                 break;
2718         case DWC3_DEVICE_EVENT_EOPF:
2719                 dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame");
2720                 break;
2721         case DWC3_DEVICE_EVENT_SOF:
2722                 dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame");
2723                 break;
2724         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2725                 dwc3_trace(trace_dwc3_gadget, "Erratic Error");
2726                 break;
2727         case DWC3_DEVICE_EVENT_CMD_CMPL:
2728                 dwc3_trace(trace_dwc3_gadget, "Command Complete");
2729                 break;
2730         case DWC3_DEVICE_EVENT_OVERFLOW:
2731                 dwc3_trace(trace_dwc3_gadget, "Overflow");
2732                 break;
2733         default:
2734                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2735         }
2736 }
2737
2738 static void dwc3_process_event_entry(struct dwc3 *dwc,
2739                 const union dwc3_event *event)
2740 {
2741         trace_dwc3_event(event->raw);
2742
2743         /* Endpoint IRQ, handle it and return early */
2744         if (event->type.is_devspec == 0) {
2745                 /* depevt */
2746                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2747         }
2748
2749         switch (event->type.type) {
2750         case DWC3_EVENT_TYPE_DEV:
2751                 dwc3_gadget_interrupt(dwc, &event->devt);
2752                 break;
2753         /* REVISIT what to do with Carkit and I2C events ? */
2754         default:
2755                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2756         }
2757 }
2758
2759 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
2760 {
2761         struct dwc3 *dwc = evt->dwc;
2762         irqreturn_t ret = IRQ_NONE;
2763         int left;
2764         u32 reg;
2765
2766         left = evt->count;
2767
2768         if (!(evt->flags & DWC3_EVENT_PENDING))
2769                 return IRQ_NONE;
2770
2771         while (left > 0) {
2772                 union dwc3_event event;
2773
2774                 event.raw = *(u32 *) (evt->buf + evt->lpos);
2775
2776                 dwc3_process_event_entry(dwc, &event);
2777
2778                 /*
2779                  * FIXME we wrap around correctly to the next entry as
2780                  * almost all entries are 4 bytes in size. There is one
2781                  * entry which has 12 bytes which is a regular entry
2782                  * followed by 8 bytes data. ATM I don't know how
2783                  * things are organized if we get next to the a
2784                  * boundary so I worry about that once we try to handle
2785                  * that.
2786                  */
2787                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2788                 left -= 4;
2789
2790                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 4);
2791         }
2792
2793         evt->count = 0;
2794         evt->flags &= ~DWC3_EVENT_PENDING;
2795         ret = IRQ_HANDLED;
2796
2797         /* Unmask interrupt */
2798         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
2799         reg &= ~DWC3_GEVNTSIZ_INTMASK;
2800         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
2801
2802         return ret;
2803 }
2804
2805 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
2806 {
2807         struct dwc3_event_buffer *evt = _evt;
2808         struct dwc3 *dwc = evt->dwc;
2809         unsigned long flags;
2810         irqreturn_t ret = IRQ_NONE;
2811
2812         spin_lock_irqsave(&dwc->lock, flags);
2813         ret = dwc3_process_event_buf(evt);
2814         spin_unlock_irqrestore(&dwc->lock, flags);
2815
2816         return ret;
2817 }
2818
2819 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
2820 {
2821         struct dwc3 *dwc = evt->dwc;
2822         u32 count;
2823         u32 reg;
2824
2825         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
2826         count &= DWC3_GEVNTCOUNT_MASK;
2827         if (!count)
2828                 return IRQ_NONE;
2829
2830         evt->count = count;
2831         evt->flags |= DWC3_EVENT_PENDING;
2832
2833         /* Mask interrupt */
2834         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
2835         reg |= DWC3_GEVNTSIZ_INTMASK;
2836         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
2837
2838         return IRQ_WAKE_THREAD;
2839 }
2840
2841 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
2842 {
2843         struct dwc3_event_buffer        *evt = _evt;
2844
2845         return dwc3_check_event_buf(evt);
2846 }
2847
2848 /**
2849  * dwc3_gadget_init - Initializes gadget related registers
2850  * @dwc: pointer to our controller context structure
2851  *
2852  * Returns 0 on success otherwise negative errno.
2853  */
2854 int dwc3_gadget_init(struct dwc3 *dwc)
2855 {
2856         int                                     ret;
2857
2858         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2859                         &dwc->ctrl_req_addr, GFP_KERNEL);
2860         if (!dwc->ctrl_req) {
2861                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2862                 ret = -ENOMEM;
2863                 goto err0;
2864         }
2865
2866         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
2867                         &dwc->ep0_trb_addr, GFP_KERNEL);
2868         if (!dwc->ep0_trb) {
2869                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2870                 ret = -ENOMEM;
2871                 goto err1;
2872         }
2873
2874         dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
2875         if (!dwc->setup_buf) {
2876                 ret = -ENOMEM;
2877                 goto err2;
2878         }
2879
2880         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2881                         DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2882                         GFP_KERNEL);
2883         if (!dwc->ep0_bounce) {
2884                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2885                 ret = -ENOMEM;
2886                 goto err3;
2887         }
2888
2889         dwc->zlp_buf = kzalloc(DWC3_ZLP_BUF_SIZE, GFP_KERNEL);
2890         if (!dwc->zlp_buf) {
2891                 ret = -ENOMEM;
2892                 goto err4;
2893         }
2894
2895         dwc->gadget.ops                 = &dwc3_gadget_ops;
2896         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
2897         dwc->gadget.sg_supported        = true;
2898         dwc->gadget.name                = "dwc3-gadget";
2899         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
2900
2901         /*
2902          * FIXME We might be setting max_speed to <SUPER, however versions
2903          * <2.20a of dwc3 have an issue with metastability (documented
2904          * elsewhere in this driver) which tells us we can't set max speed to
2905          * anything lower than SUPER.
2906          *
2907          * Because gadget.max_speed is only used by composite.c and function
2908          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
2909          * to happen so we avoid sending SuperSpeed Capability descriptor
2910          * together with our BOS descriptor as that could confuse host into
2911          * thinking we can handle super speed.
2912          *
2913          * Note that, in fact, we won't even support GetBOS requests when speed
2914          * is less than super speed because we don't have means, yet, to tell
2915          * composite.c that we are USB 2.0 + LPM ECN.
2916          */
2917         if (dwc->revision < DWC3_REVISION_220A)
2918                 dwc3_trace(trace_dwc3_gadget,
2919                                 "Changing max_speed on rev %08x\n",
2920                                 dwc->revision);
2921
2922         dwc->gadget.max_speed           = dwc->maximum_speed;
2923
2924         /*
2925          * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2926          * on ep out.
2927          */
2928         dwc->gadget.quirk_ep_out_aligned_size = true;
2929
2930         /*
2931          * REVISIT: Here we should clear all pending IRQs to be
2932          * sure we're starting from a well known location.
2933          */
2934
2935         ret = dwc3_gadget_init_endpoints(dwc);
2936         if (ret)
2937                 goto err5;
2938
2939         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2940         if (ret) {
2941                 dev_err(dwc->dev, "failed to register udc\n");
2942                 goto err5;
2943         }
2944
2945         return 0;
2946
2947 err5:
2948         kfree(dwc->zlp_buf);
2949
2950 err4:
2951         dwc3_gadget_free_endpoints(dwc);
2952         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2953                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2954
2955 err3:
2956         kfree(dwc->setup_buf);
2957
2958 err2:
2959         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2960                         dwc->ep0_trb, dwc->ep0_trb_addr);
2961
2962 err1:
2963         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2964                         dwc->ctrl_req, dwc->ctrl_req_addr);
2965
2966 err0:
2967         return ret;
2968 }
2969
2970 /* -------------------------------------------------------------------------- */
2971
2972 void dwc3_gadget_exit(struct dwc3 *dwc)
2973 {
2974         usb_del_gadget_udc(&dwc->gadget);
2975
2976         dwc3_gadget_free_endpoints(dwc);
2977
2978         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2979                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2980
2981         kfree(dwc->setup_buf);
2982         kfree(dwc->zlp_buf);
2983
2984         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2985                         dwc->ep0_trb, dwc->ep0_trb_addr);
2986
2987         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2988                         dwc->ctrl_req, dwc->ctrl_req_addr);
2989 }
2990
2991 int dwc3_gadget_suspend(struct dwc3 *dwc)
2992 {
2993         int ret;
2994
2995         if (!dwc->gadget_driver)
2996                 return 0;
2997
2998         ret = dwc3_gadget_run_stop(dwc, false, false);
2999         if (ret < 0)
3000                 return ret;
3001
3002         dwc3_disconnect_gadget(dwc);
3003         __dwc3_gadget_stop(dwc);
3004
3005         return 0;
3006 }
3007
3008 int dwc3_gadget_resume(struct dwc3 *dwc)
3009 {
3010         int                     ret;
3011
3012         if (!dwc->gadget_driver)
3013                 return 0;
3014
3015         ret = __dwc3_gadget_start(dwc);
3016         if (ret < 0)
3017                 goto err0;
3018
3019         ret = dwc3_gadget_run_stop(dwc, true, false);
3020         if (ret < 0)
3021                 goto err1;
3022
3023         return 0;
3024
3025 err1:
3026         __dwc3_gadget_stop(dwc);
3027
3028 err0:
3029         return ret;
3030 }