arch/mips/cavium-octeon/octeon-irq.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2004-2016 Cavium, Inc.
   7  */
   8
   9 #include <linux/of_address.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/irqdomain.h>
  12 #include <linux/bitops.h>
  13 #include <linux/of_irq.h>
  14 #include <linux/percpu.h>
  15 #include <linux/slab.h>
  16 #include <linux/irq.h>
  17 #include <linux/smp.h>
  18 #include <linux/of.h>
  19
  20 #include <asm/octeon/octeon.h>
  21 #include <asm/octeon/cvmx-ciu2-defs.h>
  22
  23 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);
  24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);
  25 static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock);
  26
  27 struct octeon_irq_ciu_domain_data {
  28         int num_sum;  /* number of sum registers (2 or 3). */
  29 };
  30
  31 static __read_mostly int octeon_irq_ciu_to_irq[8][64];
  32
  33 struct octeon_ciu_chip_data {
  34         union {
  35                 struct {                /* only used for ciu3 */
  36                         u64 ciu3_addr;
  37                         unsigned int intsn;
  38                 };
  39                 struct {                /* only used for ciu/ciu2 */
  40                         u8 line;
  41                         u8 bit;
  42                         u8 gpio_line;
  43                 };
  44         };
  45         int current_cpu;        /* Next CPU expected to take this irq */
  46 };
  47
  48 struct octeon_core_chip_data {
  49         struct mutex core_irq_mutex;
  50         bool current_en;
  51         bool desired_en;
  52         u8 bit;
  53 };
  54
  55 #define MIPS_CORE_IRQ_LINES 8
  56
  57 static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES];
  58
  59 static int octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line,
  60                                       struct irq_chip *chip,
  61                                       irq_flow_handler_t handler)
  62 {
  63         struct octeon_ciu_chip_data *cd;
  64
  65         cd = kzalloc(sizeof(*cd), GFP_KERNEL);
  66         if (!cd)
  67                 return -ENOMEM;
  68
  69         irq_set_chip_and_handler(irq, chip, handler);
  70
  71         cd->line = line;
  72         cd->bit = bit;
  73         cd->gpio_line = gpio_line;
  74
  75         irq_set_chip_data(irq, cd);
  76         octeon_irq_ciu_to_irq[line][bit] = irq;
  77         return 0;
  78 }
  79
  80 static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq)
  81 {
  82         struct irq_data *data = irq_get_irq_data(irq);
  83         struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data);
  84
  85         irq_set_chip_data(irq, NULL);
  86         kfree(cd);
  87 }
  88
  89 static int octeon_irq_force_ciu_mapping(struct irq_domain *domain,
  90                                         int irq, int line, int bit)
  91 {
  92         return irq_domain_associate(domain, irq, line << 6 | bit);
  93 }
  94
  95 static int octeon_coreid_for_cpu(int cpu)
  96 {
  97 #ifdef CONFIG_SMP
  98         return cpu_logical_map(cpu);
  99 #else
 100         return cvmx_get_core_num();
 101 #endif
 102 }
 103
 104 static int octeon_cpu_for_coreid(int coreid)
 105 {
 106 #ifdef CONFIG_SMP
 107         return cpu_number_map(coreid);
 108 #else
 109         return smp_processor_id();
 110 #endif
 111 }
 112
 113 static void octeon_irq_core_ack(struct irq_data *data)
 114 {
 115         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 116         unsigned int bit = cd->bit;
 117
 118         /*
 119          * We don't need to disable IRQs to make these atomic since
 120          * they are already disabled earlier in the low level
 121          * interrupt code.
 122          */
 123         clear_c0_status(0x100 << bit);
 124         /* The two user interrupts must be cleared manually. */
 125         if (bit < 2)
 126                 clear_c0_cause(0x100 << bit);
 127 }
 128
 129 static void octeon_irq_core_eoi(struct irq_data *data)
 130 {
 131         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 132
 133         /*
 134          * We don't need to disable IRQs to make these atomic since
 135          * they are already disabled earlier in the low level
 136          * interrupt code.
 137          */
 138         set_c0_status(0x100 << cd->bit);
 139 }
 140
 141 static void octeon_irq_core_set_enable_local(void *arg)
 142 {
 143         struct irq_data *data = arg;
 144         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 145         unsigned int mask = 0x100 << cd->bit;
 146
 147         /*
 148          * Interrupts are already disabled, so these are atomic.
 149          */
 150         if (cd->desired_en)
 151                 set_c0_status(mask);
 152         else
 153                 clear_c0_status(mask);
 154
 155 }
 156
 157 static void octeon_irq_core_disable(struct irq_data *data)
 158 {
 159         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 160         cd->desired_en = false;
 161 }
 162
 163 static void octeon_irq_core_enable(struct irq_data *data)
 164 {
 165         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 166         cd->desired_en = true;
 167 }
 168
 169 static void octeon_irq_core_bus_lock(struct irq_data *data)
 170 {
 171         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 172
 173         mutex_lock(&cd->core_irq_mutex);
 174 }
 175
 176 static void octeon_irq_core_bus_sync_unlock(struct irq_data *data)
 177 {
 178         struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
 179
 180         if (cd->desired_en != cd->current_en) {
 181                 on_each_cpu(octeon_irq_core_set_enable_local, data, 1);
 182
 183                 cd->current_en = cd->desired_en;
 184         }
 185
 186         mutex_unlock(&cd->core_irq_mutex);
 187 }
 188
 189 static struct irq_chip octeon_irq_chip_core = {
 190         .name = "Core",
 191         .irq_enable = octeon_irq_core_enable,
 192         .irq_disable = octeon_irq_core_disable,
 193         .irq_ack = octeon_irq_core_ack,
 194         .irq_eoi = octeon_irq_core_eoi,
 195         .irq_bus_lock = octeon_irq_core_bus_lock,
 196         .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock,
 197
 198         .irq_cpu_online = octeon_irq_core_eoi,
 199         .irq_cpu_offline = octeon_irq_core_ack,
 200         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 201 };
 202
 203 static void __init octeon_irq_init_core(void)
 204 {
 205         int i;
 206         int irq;
 207         struct octeon_core_chip_data *cd;
 208
 209         for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) {
 210                 cd = &octeon_irq_core_chip_data[i];
 211                 cd->current_en = false;
 212                 cd->desired_en = false;
 213                 cd->bit = i;
 214                 mutex_init(&cd->core_irq_mutex);
 215
 216                 irq = OCTEON_IRQ_SW0 + i;
 217                 irq_set_chip_data(irq, cd);
 218                 irq_set_chip_and_handler(irq, &octeon_irq_chip_core,
 219                                          handle_percpu_irq);
 220         }
 221 }
 222
 223 static int next_cpu_for_irq(struct irq_data *data)
 224 {
 225
 226 #ifdef CONFIG_SMP
 227         int cpu;
 228         struct cpumask *mask = irq_data_get_affinity_mask(data);
 229         int weight = cpumask_weight(mask);
 230         struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data);
 231
 232         if (weight > 1) {
 233                 cpu = cd->current_cpu;
 234                 for (;;) {
 235                         cpu = cpumask_next(cpu, mask);
 236                         if (cpu >= nr_cpu_ids) {
 237                                 cpu = -1;
 238                                 continue;
 239                         } else if (cpumask_test_cpu(cpu, cpu_online_mask)) {
 240                                 break;
 241                         }
 242                 }
 243         } else if (weight == 1) {
 244                 cpu = cpumask_first(mask);
 245         } else {
 246                 cpu = smp_processor_id();
 247         }
 248         cd->current_cpu = cpu;
 249         return cpu;
 250 #else
 251         return smp_processor_id();
 252 #endif
 253 }
 254
 255 static void octeon_irq_ciu_enable(struct irq_data *data)
 256 {
 257         int cpu = next_cpu_for_irq(data);
 258         int coreid = octeon_coreid_for_cpu(cpu);
 259         unsigned long *pen;
 260         unsigned long flags;
 261         struct octeon_ciu_chip_data *cd;
 262         raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 263
 264         cd = irq_data_get_irq_chip_data(data);
 265
 266         raw_spin_lock_irqsave(lock, flags);
 267         if (cd->line == 0) {
 268                 pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 269                 __set_bit(cd->bit, pen);
 270                 /*
 271                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 272                  * enabling the irq.
 273                  */
 274                 wmb();
 275                 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 276         } else {
 277                 pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 278                 __set_bit(cd->bit, pen);
 279                 /*
 280                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 281                  * enabling the irq.
 282                  */
 283                 wmb();
 284                 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 285         }
 286         raw_spin_unlock_irqrestore(lock, flags);
 287 }
 288
 289 static void octeon_irq_ciu_enable_local(struct irq_data *data)
 290 {
 291         unsigned long *pen;
 292         unsigned long flags;
 293         struct octeon_ciu_chip_data *cd;
 294         raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock);
 295
 296         cd = irq_data_get_irq_chip_data(data);
 297
 298         raw_spin_lock_irqsave(lock, flags);
 299         if (cd->line == 0) {
 300                 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror);
 301                 __set_bit(cd->bit, pen);
 302                 /*
 303                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 304                  * enabling the irq.
 305                  */
 306                 wmb();
 307                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
 308         } else {
 309                 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror);
 310                 __set_bit(cd->bit, pen);
 311                 /*
 312                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 313                  * enabling the irq.
 314                  */
 315                 wmb();
 316                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
 317         }
 318         raw_spin_unlock_irqrestore(lock, flags);
 319 }
 320
 321 static void octeon_irq_ciu_disable_local(struct irq_data *data)
 322 {
 323         unsigned long *pen;
 324         unsigned long flags;
 325         struct octeon_ciu_chip_data *cd;
 326         raw_spinlock_t *lock = this_cpu_ptr(&octeon_irq_ciu_spinlock);
 327
 328         cd = irq_data_get_irq_chip_data(data);
 329
 330         raw_spin_lock_irqsave(lock, flags);
 331         if (cd->line == 0) {
 332                 pen = this_cpu_ptr(&octeon_irq_ciu0_en_mirror);
 333                 __clear_bit(cd->bit, pen);
 334                 /*
 335                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 336                  * enabling the irq.
 337                  */
 338                 wmb();
 339                 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
 340         } else {
 341                 pen = this_cpu_ptr(&octeon_irq_ciu1_en_mirror);
 342                 __clear_bit(cd->bit, pen);
 343                 /*
 344                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 345                  * enabling the irq.
 346                  */
 347                 wmb();
 348                 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
 349         }
 350         raw_spin_unlock_irqrestore(lock, flags);
 351 }
 352
 353 static void octeon_irq_ciu_disable_all(struct irq_data *data)
 354 {
 355         unsigned long flags;
 356         unsigned long *pen;
 357         int cpu;
 358         struct octeon_ciu_chip_data *cd;
 359         raw_spinlock_t *lock;
 360
 361         cd = irq_data_get_irq_chip_data(data);
 362
 363         for_each_online_cpu(cpu) {
 364                 int coreid = octeon_coreid_for_cpu(cpu);
 365                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 366                 if (cd->line == 0)
 367                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 368                 else
 369                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 370
 371                 raw_spin_lock_irqsave(lock, flags);
 372                 __clear_bit(cd->bit, pen);
 373                 /*
 374                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 375                  * enabling the irq.
 376                  */
 377                 wmb();
 378                 if (cd->line == 0)
 379                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 380                 else
 381                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 382                 raw_spin_unlock_irqrestore(lock, flags);
 383         }
 384 }
 385
 386 static void octeon_irq_ciu_enable_all(struct irq_data *data)
 387 {
 388         unsigned long flags;
 389         unsigned long *pen;
 390         int cpu;
 391         struct octeon_ciu_chip_data *cd;
 392         raw_spinlock_t *lock;
 393
 394         cd = irq_data_get_irq_chip_data(data);
 395
 396         for_each_online_cpu(cpu) {
 397                 int coreid = octeon_coreid_for_cpu(cpu);
 398                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 399                 if (cd->line == 0)
 400                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 401                 else
 402                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 403
 404                 raw_spin_lock_irqsave(lock, flags);
 405                 __set_bit(cd->bit, pen);
 406                 /*
 407                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 408                  * enabling the irq.
 409                  */
 410                 wmb();
 411                 if (cd->line == 0)
 412                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 413                 else
 414                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 415                 raw_spin_unlock_irqrestore(lock, flags);
 416         }
 417 }
 418
 419 /*
 420  * Enable the irq on the next core in the affinity set for chips that
 421  * have the EN*_W1{S,C} registers.
 422  */
 423 static void octeon_irq_ciu_enable_v2(struct irq_data *data)
 424 {
 425         u64 mask;
 426         int cpu = next_cpu_for_irq(data);
 427         struct octeon_ciu_chip_data *cd;
 428
 429         cd = irq_data_get_irq_chip_data(data);
 430         mask = 1ull << (cd->bit);
 431
 432         /*
 433          * Called under the desc lock, so these should never get out
 434          * of sync.
 435          */
 436         if (cd->line == 0) {
 437                 int index = octeon_coreid_for_cpu(cpu) * 2;
 438                 set_bit(cd->bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 439                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 440         } else {
 441                 int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 442                 set_bit(cd->bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 443                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 444         }
 445 }
 446
 447 /*
 448  * Enable the irq in the sum2 registers.
 449  */
 450 static void octeon_irq_ciu_enable_sum2(struct irq_data *data)
 451 {
 452         u64 mask;
 453         int cpu = next_cpu_for_irq(data);
 454         int index = octeon_coreid_for_cpu(cpu);
 455         struct octeon_ciu_chip_data *cd;
 456
 457         cd = irq_data_get_irq_chip_data(data);
 458         mask = 1ull << (cd->bit);
 459
 460         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask);
 461 }
 462
 463 /*
 464  * Disable the irq in the sum2 registers.
 465  */
 466 static void octeon_irq_ciu_disable_local_sum2(struct irq_data *data)
 467 {
 468         u64 mask;
 469         int cpu = next_cpu_for_irq(data);
 470         int index = octeon_coreid_for_cpu(cpu);
 471         struct octeon_ciu_chip_data *cd;
 472
 473         cd = irq_data_get_irq_chip_data(data);
 474         mask = 1ull << (cd->bit);
 475
 476         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask);
 477 }
 478
 479 static void octeon_irq_ciu_ack_sum2(struct irq_data *data)
 480 {
 481         u64 mask;
 482         int cpu = next_cpu_for_irq(data);
 483         int index = octeon_coreid_for_cpu(cpu);
 484         struct octeon_ciu_chip_data *cd;
 485
 486         cd = irq_data_get_irq_chip_data(data);
 487         mask = 1ull << (cd->bit);
 488
 489         cvmx_write_csr(CVMX_CIU_SUM2_PPX_IP4(index), mask);
 490 }
 491
 492 static void octeon_irq_ciu_disable_all_sum2(struct irq_data *data)
 493 {
 494         int cpu;
 495         struct octeon_ciu_chip_data *cd;
 496         u64 mask;
 497
 498         cd = irq_data_get_irq_chip_data(data);
 499         mask = 1ull << (cd->bit);
 500
 501         for_each_online_cpu(cpu) {
 502                 int coreid = octeon_coreid_for_cpu(cpu);
 503
 504                 cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(coreid), mask);
 505         }
 506 }
 507
 508 /*
 509  * Enable the irq on the current CPU for chips that
 510  * have the EN*_W1{S,C} registers.
 511  */
 512 static void octeon_irq_ciu_enable_local_v2(struct irq_data *data)
 513 {
 514         u64 mask;
 515         struct octeon_ciu_chip_data *cd;
 516
 517         cd = irq_data_get_irq_chip_data(data);
 518         mask = 1ull << (cd->bit);
 519
 520         if (cd->line == 0) {
 521                 int index = cvmx_get_core_num() * 2;
 522                 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror));
 523                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 524         } else {
 525                 int index = cvmx_get_core_num() * 2 + 1;
 526                 set_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror));
 527                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 528         }
 529 }
 530
 531 static void octeon_irq_ciu_disable_local_v2(struct irq_data *data)
 532 {
 533         u64 mask;
 534         struct octeon_ciu_chip_data *cd;
 535
 536         cd = irq_data_get_irq_chip_data(data);
 537         mask = 1ull << (cd->bit);
 538
 539         if (cd->line == 0) {
 540                 int index = cvmx_get_core_num() * 2;
 541                 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu0_en_mirror));
 542                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 543         } else {
 544                 int index = cvmx_get_core_num() * 2 + 1;
 545                 clear_bit(cd->bit, this_cpu_ptr(&octeon_irq_ciu1_en_mirror));
 546                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 547         }
 548 }
 549
 550 /*
 551  * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.
 552  */
 553 static void octeon_irq_ciu_ack(struct irq_data *data)
 554 {
 555         u64 mask;
 556         struct octeon_ciu_chip_data *cd;
 557
 558         cd = irq_data_get_irq_chip_data(data);
 559         mask = 1ull << (cd->bit);
 560
 561         if (cd->line == 0) {
 562                 int index = cvmx_get_core_num() * 2;
 563                 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
 564         } else {
 565                 cvmx_write_csr(CVMX_CIU_INT_SUM1, mask);
 566         }
 567 }
 568
 569 /*
 570  * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
 571  * registers.
 572  */
 573 static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)
 574 {
 575         int cpu;
 576         u64 mask;
 577         struct octeon_ciu_chip_data *cd;
 578
 579         cd = irq_data_get_irq_chip_data(data);
 580         mask = 1ull << (cd->bit);
 581
 582         if (cd->line == 0) {
 583                 for_each_online_cpu(cpu) {
 584                         int index = octeon_coreid_for_cpu(cpu) * 2;
 585                         clear_bit(cd->bit,
 586                                 &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 587                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 588                 }
 589         } else {
 590                 for_each_online_cpu(cpu) {
 591                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 592                         clear_bit(cd->bit,
 593                                 &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 594                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 595                 }
 596         }
 597 }
 598
 599 /*
 600  * Enable the irq on the all cores for chips that have the EN*_W1{S,C}
 601  * registers.
 602  */
 603 static void octeon_irq_ciu_enable_all_v2(struct irq_data *data)
 604 {
 605         int cpu;
 606         u64 mask;
 607         struct octeon_ciu_chip_data *cd;
 608
 609         cd = irq_data_get_irq_chip_data(data);
 610         mask = 1ull << (cd->bit);
 611
 612         if (cd->line == 0) {
 613                 for_each_online_cpu(cpu) {
 614                         int index = octeon_coreid_for_cpu(cpu) * 2;
 615                         set_bit(cd->bit,
 616                                 &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
 617                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 618                 }
 619         } else {
 620                 for_each_online_cpu(cpu) {
 621                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 622                         set_bit(cd->bit,
 623                                 &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
 624                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 625                 }
 626         }
 627 }
 628
 629 static void octeon_irq_gpio_setup(struct irq_data *data)
 630 {
 631         union cvmx_gpio_bit_cfgx cfg;
 632         struct octeon_ciu_chip_data *cd;
 633         u32 t = irqd_get_trigger_type(data);
 634
 635         cd = irq_data_get_irq_chip_data(data);
 636
 637         cfg.u64 = 0;
 638         cfg.s.int_en = 1;
 639         cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0;
 640         cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING)) != 0;
 641
 642         /* 140 nS glitch filter*/
 643         cfg.s.fil_cnt = 7;
 644         cfg.s.fil_sel = 3;
 645
 646         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), cfg.u64);
 647 }
 648
 649 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data)
 650 {
 651         octeon_irq_gpio_setup(data);
 652         octeon_irq_ciu_enable_v2(data);
 653 }
 654
 655 static void octeon_irq_ciu_enable_gpio(struct irq_data *data)
 656 {
 657         octeon_irq_gpio_setup(data);
 658         octeon_irq_ciu_enable(data);
 659 }
 660
 661 static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t)
 662 {
 663         irqd_set_trigger_type(data, t);
 664         octeon_irq_gpio_setup(data);
 665
 666         if (irqd_get_trigger_type(data) & IRQ_TYPE_EDGE_BOTH)
 667                 irq_set_handler_locked(data, handle_edge_irq);
 668         else
 669                 irq_set_handler_locked(data, handle_level_irq);
 670
 671         return IRQ_SET_MASK_OK;
 672 }
 673
 674 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data)
 675 {
 676         struct octeon_ciu_chip_data *cd;
 677
 678         cd = irq_data_get_irq_chip_data(data);
 679         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
 680
 681         octeon_irq_ciu_disable_all_v2(data);
 682 }
 683
 684 static void octeon_irq_ciu_disable_gpio(struct irq_data *data)
 685 {
 686         struct octeon_ciu_chip_data *cd;
 687
 688         cd = irq_data_get_irq_chip_data(data);
 689         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
 690
 691         octeon_irq_ciu_disable_all(data);
 692 }
 693
 694 static void octeon_irq_ciu_gpio_ack(struct irq_data *data)
 695 {
 696         struct octeon_ciu_chip_data *cd;
 697         u64 mask;
 698
 699         cd = irq_data_get_irq_chip_data(data);
 700         mask = 1ull << (cd->gpio_line);
 701
 702         cvmx_write_csr(CVMX_GPIO_INT_CLR, mask);
 703 }
 704
 705 #ifdef CONFIG_SMP
 706
 707 static void octeon_irq_cpu_offline_ciu(struct irq_data *data)
 708 {
 709         int cpu = smp_processor_id();
 710         cpumask_t new_affinity;
 711         struct cpumask *mask = irq_data_get_affinity_mask(data);
 712
 713         if (!cpumask_test_cpu(cpu, mask))
 714                 return;
 715
 716         if (cpumask_weight(mask) > 1) {
 717                 /*
 718                  * It has multi CPU affinity, just remove this CPU
 719                  * from the affinity set.
 720                  */
 721                 cpumask_copy(&new_affinity, mask);
 722                 cpumask_clear_cpu(cpu, &new_affinity);
 723         } else {
 724                 /* Otherwise, put it on lowest numbered online CPU. */
 725                 cpumask_clear(&new_affinity);
 726                 cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);
 727         }
 728         irq_set_affinity_locked(data, &new_affinity, false);
 729 }
 730
 731 static int octeon_irq_ciu_set_affinity(struct irq_data *data,
 732                                        const struct cpumask *dest, bool force)
 733 {
 734         int cpu;
 735         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 736         unsigned long flags;
 737         struct octeon_ciu_chip_data *cd;
 738         unsigned long *pen;
 739         raw_spinlock_t *lock;
 740
 741         cd = irq_data_get_irq_chip_data(data);
 742
 743         /*
 744          * For non-v2 CIU, we will allow only single CPU affinity.
 745          * This removes the need to do locking in the .ack/.eoi
 746          * functions.
 747          */
 748         if (cpumask_weight(dest) != 1)
 749                 return -EINVAL;
 750
 751         if (!enable_one)
 752                 return 0;
 753
 754
 755         for_each_online_cpu(cpu) {
 756                 int coreid = octeon_coreid_for_cpu(cpu);
 757
 758                 lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 759                 raw_spin_lock_irqsave(lock, flags);
 760
 761                 if (cd->line == 0)
 762                         pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 763                 else
 764                         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 765
 766                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
 767                         enable_one = 0;
 768                         __set_bit(cd->bit, pen);
 769                 } else {
 770                         __clear_bit(cd->bit, pen);
 771                 }
 772                 /*
 773                  * Must be visible to octeon_irq_ip{2,3}_ciu() before
 774                  * enabling the irq.
 775                  */
 776                 wmb();
 777
 778                 if (cd->line == 0)
 779                         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
 780                 else
 781                         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
 782
 783                 raw_spin_unlock_irqrestore(lock, flags);
 784         }
 785         return 0;
 786 }
 787
 788 /*
 789  * Set affinity for the irq for chips that have the EN*_W1{S,C}
 790  * registers.
 791  */
 792 static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,
 793                                           const struct cpumask *dest,
 794                                           bool force)
 795 {
 796         int cpu;
 797         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 798         u64 mask;
 799         struct octeon_ciu_chip_data *cd;
 800
 801         if (!enable_one)
 802                 return 0;
 803
 804         cd = irq_data_get_irq_chip_data(data);
 805         mask = 1ull << cd->bit;
 806
 807         if (cd->line == 0) {
 808                 for_each_online_cpu(cpu) {
 809                         unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 810                         int index = octeon_coreid_for_cpu(cpu) * 2;
 811                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 812                                 enable_one = false;
 813                                 set_bit(cd->bit, pen);
 814                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 815                         } else {
 816                                 clear_bit(cd->bit, pen);
 817                                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 818                         }
 819                 }
 820         } else {
 821                 for_each_online_cpu(cpu) {
 822                         unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
 823                         int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 824                         if (cpumask_test_cpu(cpu, dest) && enable_one) {
 825                                 enable_one = false;
 826                                 set_bit(cd->bit, pen);
 827                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 828                         } else {
 829                                 clear_bit(cd->bit, pen);
 830                                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 831                         }
 832                 }
 833         }
 834         return 0;
 835 }
 836
 837 static int octeon_irq_ciu_set_affinity_sum2(struct irq_data *data,
 838                                             const struct cpumask *dest,
 839                                             bool force)
 840 {
 841         int cpu;
 842         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 843         u64 mask;
 844         struct octeon_ciu_chip_data *cd;
 845
 846         if (!enable_one)
 847                 return 0;
 848
 849         cd = irq_data_get_irq_chip_data(data);
 850         mask = 1ull << cd->bit;
 851
 852         for_each_online_cpu(cpu) {
 853                 int index = octeon_coreid_for_cpu(cpu);
 854
 855                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
 856                         enable_one = false;
 857                         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1S(index), mask);
 858                 } else {
 859                         cvmx_write_csr(CVMX_CIU_EN2_PPX_IP4_W1C(index), mask);
 860                 }
 861         }
 862         return 0;
 863 }
 864 #endif
 865
 866 /*
 867  * Newer octeon chips have support for lockless CIU operation.
 868  */
 869 static struct irq_chip octeon_irq_chip_ciu_v2 = {
 870         .name = "CIU",
 871         .irq_enable = octeon_irq_ciu_enable_v2,
 872         .irq_disable = octeon_irq_ciu_disable_all_v2,
 873         .irq_mask = octeon_irq_ciu_disable_local_v2,
 874         .irq_unmask = octeon_irq_ciu_enable_v2,
 875 #ifdef CONFIG_SMP
 876         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 877         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 878 #endif
 879 };
 880
 881 static struct irq_chip octeon_irq_chip_ciu_v2_edge = {
 882         .name = "CIU",
 883         .irq_enable = octeon_irq_ciu_enable_v2,
 884         .irq_disable = octeon_irq_ciu_disable_all_v2,
 885         .irq_ack = octeon_irq_ciu_ack,
 886         .irq_mask = octeon_irq_ciu_disable_local_v2,
 887         .irq_unmask = octeon_irq_ciu_enable_v2,
 888 #ifdef CONFIG_SMP
 889         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 890         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 891 #endif
 892 };
 893
 894 /*
 895  * Newer octeon chips have support for lockless CIU operation.
 896  */
 897 static struct irq_chip octeon_irq_chip_ciu_sum2 = {
 898         .name = "CIU",
 899         .irq_enable = octeon_irq_ciu_enable_sum2,
 900         .irq_disable = octeon_irq_ciu_disable_all_sum2,
 901         .irq_mask = octeon_irq_ciu_disable_local_sum2,
 902         .irq_unmask = octeon_irq_ciu_enable_sum2,
 903 #ifdef CONFIG_SMP
 904         .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2,
 905         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 906 #endif
 907 };
 908
 909 static struct irq_chip octeon_irq_chip_ciu_sum2_edge = {
 910         .name = "CIU",
 911         .irq_enable = octeon_irq_ciu_enable_sum2,
 912         .irq_disable = octeon_irq_ciu_disable_all_sum2,
 913         .irq_ack = octeon_irq_ciu_ack_sum2,
 914         .irq_mask = octeon_irq_ciu_disable_local_sum2,
 915         .irq_unmask = octeon_irq_ciu_enable_sum2,
 916 #ifdef CONFIG_SMP
 917         .irq_set_affinity = octeon_irq_ciu_set_affinity_sum2,
 918         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 919 #endif
 920 };
 921
 922 static struct irq_chip octeon_irq_chip_ciu = {
 923         .name = "CIU",
 924         .irq_enable = octeon_irq_ciu_enable,
 925         .irq_disable = octeon_irq_ciu_disable_all,
 926         .irq_mask = octeon_irq_ciu_disable_local,
 927         .irq_unmask = octeon_irq_ciu_enable,
 928 #ifdef CONFIG_SMP
 929         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 930         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 931 #endif
 932 };
 933
 934 static struct irq_chip octeon_irq_chip_ciu_edge = {
 935         .name = "CIU",
 936         .irq_enable = octeon_irq_ciu_enable,
 937         .irq_disable = octeon_irq_ciu_disable_all,
 938         .irq_ack = octeon_irq_ciu_ack,
 939         .irq_mask = octeon_irq_ciu_disable_local,
 940         .irq_unmask = octeon_irq_ciu_enable,
 941 #ifdef CONFIG_SMP
 942         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 943         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 944 #endif
 945 };
 946
 947 /* The mbox versions don't do any affinity or round-robin. */
 948 static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = {
 949         .name = "CIU-M",
 950         .irq_enable = octeon_irq_ciu_enable_all_v2,
 951         .irq_disable = octeon_irq_ciu_disable_all_v2,
 952         .irq_ack = octeon_irq_ciu_disable_local_v2,
 953         .irq_eoi = octeon_irq_ciu_enable_local_v2,
 954
 955         .irq_cpu_online = octeon_irq_ciu_enable_local_v2,
 956         .irq_cpu_offline = octeon_irq_ciu_disable_local_v2,
 957         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 958 };
 959
 960 static struct irq_chip octeon_irq_chip_ciu_mbox = {
 961         .name = "CIU-M",
 962         .irq_enable = octeon_irq_ciu_enable_all,
 963         .irq_disable = octeon_irq_ciu_disable_all,
 964         .irq_ack = octeon_irq_ciu_disable_local,
 965         .irq_eoi = octeon_irq_ciu_enable_local,
 966
 967         .irq_cpu_online = octeon_irq_ciu_enable_local,
 968         .irq_cpu_offline = octeon_irq_ciu_disable_local,
 969         .flags = IRQCHIP_ONOFFLINE_ENABLED,
 970 };
 971
 972 static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = {
 973         .name = "CIU-GPIO",
 974         .irq_enable = octeon_irq_ciu_enable_gpio_v2,
 975         .irq_disable = octeon_irq_ciu_disable_gpio_v2,
 976         .irq_ack = octeon_irq_ciu_gpio_ack,
 977         .irq_mask = octeon_irq_ciu_disable_local_v2,
 978         .irq_unmask = octeon_irq_ciu_enable_v2,
 979         .irq_set_type = octeon_irq_ciu_gpio_set_type,
 980 #ifdef CONFIG_SMP
 981         .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
 982         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 983 #endif
 984         .flags = IRQCHIP_SET_TYPE_MASKED,
 985 };
 986
 987 static struct irq_chip octeon_irq_chip_ciu_gpio = {
 988         .name = "CIU-GPIO",
 989         .irq_enable = octeon_irq_ciu_enable_gpio,
 990         .irq_disable = octeon_irq_ciu_disable_gpio,
 991         .irq_mask = octeon_irq_ciu_disable_local,
 992         .irq_unmask = octeon_irq_ciu_enable,
 993         .irq_ack = octeon_irq_ciu_gpio_ack,
 994         .irq_set_type = octeon_irq_ciu_gpio_set_type,
 995 #ifdef CONFIG_SMP
 996         .irq_set_affinity = octeon_irq_ciu_set_affinity,
 997         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
 998 #endif
 999         .flags = IRQCHIP_SET_TYPE_MASKED,
1000 };
1001
1002 /*
1003  * Watchdog interrupts are special.  They are associated with a single
1004  * core, so we hardwire the affinity to that core.
1005  */
1006 static void octeon_irq_ciu_wd_enable(struct irq_data *data)
1007 {
1008         unsigned long flags;
1009         unsigned long *pen;
1010         int coreid = data->irq - OCTEON_IRQ_WDOG0;      /* Bit 0-63 of EN1 */
1011         int cpu = octeon_cpu_for_coreid(coreid);
1012         raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
1013
1014         raw_spin_lock_irqsave(lock, flags);
1015         pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
1016         __set_bit(coreid, pen);
1017         /*
1018          * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling
1019          * the irq.
1020          */
1021         wmb();
1022         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
1023         raw_spin_unlock_irqrestore(lock, flags);
1024 }
1025
1026 /*
1027  * Watchdog interrupts are special.  They are associated with a single
1028  * core, so we hardwire the affinity to that core.
1029  */
1030 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data)
1031 {
1032         int coreid = data->irq - OCTEON_IRQ_WDOG0;
1033         int cpu = octeon_cpu_for_coreid(coreid);
1034
1035         set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
1036         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid);
1037 }
1038
1039
1040 static struct irq_chip octeon_irq_chip_ciu_wd_v2 = {
1041         .name = "CIU-W",
1042         .irq_enable = octeon_irq_ciu1_wd_enable_v2,
1043         .irq_disable = octeon_irq_ciu_disable_all_v2,
1044         .irq_mask = octeon_irq_ciu_disable_local_v2,
1045         .irq_unmask = octeon_irq_ciu_enable_local_v2,
1046 };
1047
1048 static struct irq_chip octeon_irq_chip_ciu_wd = {
1049         .name = "CIU-W",
1050         .irq_enable = octeon_irq_ciu_wd_enable,
1051         .irq_disable = octeon_irq_ciu_disable_all,
1052         .irq_mask = octeon_irq_ciu_disable_local,
1053         .irq_unmask = octeon_irq_ciu_enable_local,
1054 };
1055
1056 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit)
1057 {
1058         bool edge = false;
1059
1060         if (line == 0)
1061                 switch (bit) {
1062                 case 48 ... 49: /* GMX DRP */
1063                 case 50: /* IPD_DRP */
1064                 case 52 ... 55: /* Timers */
1065                 case 58: /* MPI */
1066                         edge = true;
1067                         break;
1068                 default:
1069                         break;
1070                 }
1071         else /* line == 1 */
1072                 switch (bit) {
1073                 case 47: /* PTP */
1074                         edge = true;
1075                         break;
1076                 default:
1077                         break;
1078                 }
1079         return edge;
1080 }
1081
1082 struct octeon_irq_gpio_domain_data {
1083         unsigned int base_hwirq;
1084 };
1085
1086 static int octeon_irq_gpio_xlat(struct irq_domain *d,
1087                                 struct device_node *node,
1088                                 const u32 *intspec,
1089                                 unsigned int intsize,
1090                                 unsigned long *out_hwirq,
1091                                 unsigned int *out_type)
1092 {
1093         unsigned int type;
1094         unsigned int pin;
1095         unsigned int trigger;
1096
1097         if (irq_domain_get_of_node(d) != node)
1098                 return -EINVAL;
1099
1100         if (intsize < 2)
1101                 return -EINVAL;
1102
1103         pin = intspec[0];
1104         if (pin >= 16)
1105                 return -EINVAL;
1106
1107         trigger = intspec[1];
1108
1109         switch (trigger) {
1110         case 1:
1111                 type = IRQ_TYPE_EDGE_RISING;
1112                 break;
1113         case 2:
1114                 type = IRQ_TYPE_EDGE_FALLING;
1115                 break;
1116         case 4:
1117                 type = IRQ_TYPE_LEVEL_HIGH;
1118                 break;
1119         case 8:
1120                 type = IRQ_TYPE_LEVEL_LOW;
1121                 break;
1122         default:
1123                 pr_err("Error: (%s) Invalid irq trigger specification: %x\n",
1124                        node->name,
1125                        trigger);
1126                 type = IRQ_TYPE_LEVEL_LOW;
1127                 break;
1128         }
1129         *out_type = type;
1130         *out_hwirq = pin;
1131
1132         return 0;
1133 }
1134
1135 static int octeon_irq_ciu_xlat(struct irq_domain *d,
1136                                struct device_node *node,
1137                                const u32 *intspec,
1138                                unsigned int intsize,
1139                                unsigned long *out_hwirq,
1140                                unsigned int *out_type)
1141 {
1142         unsigned int ciu, bit;
1143         struct octeon_irq_ciu_domain_data *dd = d->host_data;
1144
1145         ciu = intspec[0];
1146         bit = intspec[1];
1147
1148         if (ciu >= dd->num_sum || bit > 63)
1149                 return -EINVAL;
1150
1151         *out_hwirq = (ciu << 6) | bit;
1152         *out_type = 0;
1153
1154         return 0;
1155 }
1156
1157 static struct irq_chip *octeon_irq_ciu_chip;
1158 static struct irq_chip *octeon_irq_ciu_chip_edge;
1159 static struct irq_chip *octeon_irq_gpio_chip;
1160
1161 static int octeon_irq_ciu_map(struct irq_domain *d,
1162                               unsigned int virq, irq_hw_number_t hw)
1163 {
1164         int rv;
1165         unsigned int line = hw >> 6;
1166         unsigned int bit = hw & 63;
1167         struct octeon_irq_ciu_domain_data *dd = d->host_data;
1168
1169         if (line >= dd->num_sum || octeon_irq_ciu_to_irq[line][bit] != 0)
1170                 return -EINVAL;
1171
1172         if (line == 2) {
1173                 if (octeon_irq_ciu_is_edge(line, bit))
1174                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1175                                 &octeon_irq_chip_ciu_sum2_edge,
1176                                 handle_edge_irq);
1177                 else
1178                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1179                                 &octeon_irq_chip_ciu_sum2,
1180                                 handle_level_irq);
1181         } else {
1182                 if (octeon_irq_ciu_is_edge(line, bit))
1183                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1184                                 octeon_irq_ciu_chip_edge,
1185                                 handle_edge_irq);
1186                 else
1187                         rv = octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1188                                 octeon_irq_ciu_chip,
1189                                 handle_level_irq);
1190         }
1191         return rv;
1192 }
1193
1194 static int octeon_irq_gpio_map(struct irq_domain *d,
1195                                unsigned int virq, irq_hw_number_t hw)
1196 {
1197         struct octeon_irq_gpio_domain_data *gpiod = d->host_data;
1198         unsigned int line, bit;
1199         int r;
1200
1201         line = (hw + gpiod->base_hwirq) >> 6;
1202         bit = (hw + gpiod->base_hwirq) & 63;
1203         if (line > ARRAY_SIZE(octeon_irq_ciu_to_irq) ||
1204                 octeon_irq_ciu_to_irq[line][bit] != 0)
1205                 return -EINVAL;
1206
1207         /*
1208          * Default to handle_level_irq. If the DT contains a different
1209          * trigger type, it will call the irq_set_type callback and
1210          * the handler gets updated.
1211          */
1212         r = octeon_irq_set_ciu_mapping(virq, line, bit, hw,
1213                                        octeon_irq_gpio_chip, handle_level_irq);
1214         return r;
1215 }
1216
1217 static struct irq_domain_ops octeon_irq_domain_ciu_ops = {
1218         .map = octeon_irq_ciu_map,
1219         .unmap = octeon_irq_free_cd,
1220         .xlate = octeon_irq_ciu_xlat,
1221 };
1222
1223 static struct irq_domain_ops octeon_irq_domain_gpio_ops = {
1224         .map = octeon_irq_gpio_map,
1225         .unmap = octeon_irq_free_cd,
1226         .xlate = octeon_irq_gpio_xlat,
1227 };
1228
1229 static void octeon_irq_ip2_ciu(void)
1230 {
1231         const unsigned long core_id = cvmx_get_core_num();
1232         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
1233
1234         ciu_sum &= __this_cpu_read(octeon_irq_ciu0_en_mirror);
1235         if (likely(ciu_sum)) {
1236                 int bit = fls64(ciu_sum) - 1;
1237                 int irq = octeon_irq_ciu_to_irq[0][bit];
1238                 if (likely(irq))
1239                         do_IRQ(irq);
1240                 else
1241                         spurious_interrupt();
1242         } else {
1243                 spurious_interrupt();
1244         }
1245 }
1246
1247 static void octeon_irq_ip3_ciu(void)
1248 {
1249         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
1250
1251         ciu_sum &= __this_cpu_read(octeon_irq_ciu1_en_mirror);
1252         if (likely(ciu_sum)) {
1253                 int bit = fls64(ciu_sum) - 1;
1254                 int irq = octeon_irq_ciu_to_irq[1][bit];
1255                 if (likely(irq))
1256                         do_IRQ(irq);
1257                 else
1258                         spurious_interrupt();
1259         } else {
1260                 spurious_interrupt();
1261         }
1262 }
1263
1264 static void octeon_irq_ip4_ciu(void)
1265 {
1266         int coreid = cvmx_get_core_num();
1267         u64 ciu_sum = cvmx_read_csr(CVMX_CIU_SUM2_PPX_IP4(coreid));
1268         u64 ciu_en = cvmx_read_csr(CVMX_CIU_EN2_PPX_IP4(coreid));
1269
1270         ciu_sum &= ciu_en;
1271         if (likely(ciu_sum)) {
1272                 int bit = fls64(ciu_sum) - 1;
1273                 int irq = octeon_irq_ciu_to_irq[2][bit];
1274
1275                 if (likely(irq))
1276                         do_IRQ(irq);
1277                 else
1278                         spurious_interrupt();
1279         } else {
1280                 spurious_interrupt();
1281         }
1282 }
1283
1284 static bool octeon_irq_use_ip4;
1285
1286 static void octeon_irq_local_enable_ip4(void *arg)
1287 {
1288         set_c0_status(STATUSF_IP4);
1289 }
1290
1291 static void octeon_irq_ip4_mask(void)
1292 {
1293         clear_c0_status(STATUSF_IP4);
1294         spurious_interrupt();
1295 }
1296
1297 static void (*octeon_irq_ip2)(void);
1298 static void (*octeon_irq_ip3)(void);
1299 static void (*octeon_irq_ip4)(void);
1300
1301 void (*octeon_irq_setup_secondary)(void);
1302
1303 void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h)
1304 {
1305         octeon_irq_ip4 = h;
1306         octeon_irq_use_ip4 = true;
1307         on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1);
1308 }
1309
1310 static void octeon_irq_percpu_enable(void)
1311 {
1312         irq_cpu_online();
1313 }
1314
1315 static void octeon_irq_init_ciu_percpu(void)
1316 {
1317         int coreid = cvmx_get_core_num();
1318
1319
1320         __this_cpu_write(octeon_irq_ciu0_en_mirror, 0);
1321         __this_cpu_write(octeon_irq_ciu1_en_mirror, 0);
1322         wmb();
1323         raw_spin_lock_init(this_cpu_ptr(&octeon_irq_ciu_spinlock));
1324         /*
1325          * Disable All CIU Interrupts. The ones we need will be
1326          * enabled later.  Read the SUM register so we know the write
1327          * completed.
1328          */
1329         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0);
1330         cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0);
1331         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0);
1332         cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0);
1333         cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2)));
1334 }
1335
1336 static void octeon_irq_init_ciu2_percpu(void)
1337 {
1338         u64 regx, ipx;
1339         int coreid = cvmx_get_core_num();
1340         u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid);
1341
1342         /*
1343          * Disable All CIU2 Interrupts. The ones we need will be
1344          * enabled later.  Read the SUM register so we know the write
1345          * completed.
1346          *
1347          * There are 9 registers and 3 IPX levels with strides 0x1000
1348          * and 0x200 respectivly.  Use loops to clear them.
1349          */
1350         for (regx = 0; regx <= 0x8000; regx += 0x1000) {
1351                 for (ipx = 0; ipx <= 0x400; ipx += 0x200)
1352                         cvmx_write_csr(base + regx + ipx, 0);
1353         }
1354
1355         cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid));
1356 }
1357
1358 static void octeon_irq_setup_secondary_ciu(void)
1359 {
1360         octeon_irq_init_ciu_percpu();
1361         octeon_irq_percpu_enable();
1362
1363         /* Enable the CIU lines */
1364         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1365         if (octeon_irq_use_ip4)
1366                 set_c0_status(STATUSF_IP4);
1367         else
1368                 clear_c0_status(STATUSF_IP4);
1369 }
1370
1371 static void octeon_irq_setup_secondary_ciu2(void)
1372 {
1373         octeon_irq_init_ciu2_percpu();
1374         octeon_irq_percpu_enable();
1375
1376         /* Enable the CIU lines */
1377         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1378         if (octeon_irq_use_ip4)
1379                 set_c0_status(STATUSF_IP4);
1380         else
1381                 clear_c0_status(STATUSF_IP4);
1382 }
1383
1384 static int __init octeon_irq_init_ciu(
1385         struct device_node *ciu_node, struct device_node *parent)
1386 {
1387         unsigned int i, r;
1388         struct irq_chip *chip;
1389         struct irq_chip *chip_edge;
1390         struct irq_chip *chip_mbox;
1391         struct irq_chip *chip_wd;
1392         struct irq_domain *ciu_domain = NULL;
1393         struct octeon_irq_ciu_domain_data *dd;
1394
1395         dd = kzalloc(sizeof(*dd), GFP_KERNEL);
1396         if (!dd)
1397                 return -ENOMEM;
1398
1399         octeon_irq_init_ciu_percpu();
1400         octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;
1401
1402         octeon_irq_ip2 = octeon_irq_ip2_ciu;
1403         octeon_irq_ip3 = octeon_irq_ip3_ciu;
1404         if ((OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3())
1405                 && !OCTEON_IS_MODEL(OCTEON_CN63XX)) {
1406                 octeon_irq_ip4 =  octeon_irq_ip4_ciu;
1407                 dd->num_sum = 3;
1408                 octeon_irq_use_ip4 = true;
1409         } else {
1410                 octeon_irq_ip4 = octeon_irq_ip4_mask;
1411                 dd->num_sum = 2;
1412                 octeon_irq_use_ip4 = false;
1413         }
1414         if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
1415             OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
1416             OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) ||
1417             OCTEON_IS_OCTEON2() || OCTEON_IS_OCTEON3()) {
1418                 chip = &octeon_irq_chip_ciu_v2;
1419                 chip_edge = &octeon_irq_chip_ciu_v2_edge;
1420                 chip_mbox = &octeon_irq_chip_ciu_mbox_v2;
1421                 chip_wd = &octeon_irq_chip_ciu_wd_v2;
1422                 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2;
1423         } else {
1424                 chip = &octeon_irq_chip_ciu;
1425                 chip_edge = &octeon_irq_chip_ciu_edge;
1426                 chip_mbox = &octeon_irq_chip_ciu_mbox;
1427                 chip_wd = &octeon_irq_chip_ciu_wd;
1428                 octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio;
1429         }
1430         octeon_irq_ciu_chip = chip;
1431         octeon_irq_ciu_chip_edge = chip_edge;
1432
1433         /* Mips internal */
1434         octeon_irq_init_core();
1435
1436         ciu_domain = irq_domain_add_tree(
1437                 ciu_node, &octeon_irq_domain_ciu_ops, dd);
1438         irq_set_default_host(ciu_domain);
1439
1440         /* CIU_0 */
1441         for (i = 0; i < 16; i++) {
1442                 r = octeon_irq_force_ciu_mapping(
1443                         ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0);
1444                 if (r)
1445                         goto err;
1446         }
1447
1448         r = octeon_irq_set_ciu_mapping(
1449                 OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq);
1450         if (r)
1451                 goto err;
1452         r = octeon_irq_set_ciu_mapping(
1453                 OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq);
1454         if (r)
1455                 goto err;
1456
1457         for (i = 0; i < 4; i++) {
1458                 r = octeon_irq_force_ciu_mapping(
1459                         ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36);
1460                 if (r)
1461                         goto err;
1462         }
1463         for (i = 0; i < 4; i++) {
1464                 r = octeon_irq_force_ciu_mapping(
1465                         ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40);
1466                 if (r)
1467                         goto err;
1468         }
1469
1470         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI, 0, 45);
1471         if (r)
1472                 goto err;
1473
1474         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46);
1475         if (r)
1476                 goto err;
1477
1478         for (i = 0; i < 4; i++) {
1479                 r = octeon_irq_force_ciu_mapping(
1480                         ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52);
1481                 if (r)
1482                         goto err;
1483         }
1484
1485         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 0, 56);
1486         if (r)
1487                 goto err;
1488
1489         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_TWSI2, 0, 59);
1490         if (r)
1491                 goto err;
1492
1493         /* CIU_1 */
1494         for (i = 0; i < 16; i++) {
1495                 r = octeon_irq_set_ciu_mapping(
1496                         i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd,
1497                         handle_level_irq);
1498                 if (r)
1499                         goto err;
1500         }
1501
1502         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB1, 1, 17);
1503         if (r)
1504                 goto err;
1505
1506         /* Enable the CIU lines */
1507         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
1508         if (octeon_irq_use_ip4)
1509                 set_c0_status(STATUSF_IP4);
1510         else
1511                 clear_c0_status(STATUSF_IP4);
1512
1513         return 0;
1514 err:
1515         return r;
1516 }
1517
1518 static int __init octeon_irq_init_gpio(
1519         struct device_node *gpio_node, struct device_node *parent)
1520 {
1521         struct octeon_irq_gpio_domain_data *gpiod;
1522         u32 interrupt_cells;
1523         unsigned int base_hwirq;
1524         int r;
1525
1526         r = of_property_read_u32(parent, "#interrupt-cells", &interrupt_cells);
1527         if (r)
1528                 return r;
1529
1530         if (interrupt_cells == 1) {
1531                 u32 v;
1532
1533                 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v);
1534                 if (r) {
1535                         pr_warn("No \"interrupts\" property.\n");
1536                         return r;
1537                 }
1538                 base_hwirq = v;
1539         } else if (interrupt_cells == 2) {
1540                 u32 v0, v1;
1541
1542                 r = of_property_read_u32_index(gpio_node, "interrupts", 0, &v0);
1543                 if (r) {
1544                         pr_warn("No \"interrupts\" property.\n");
1545                         return r;
1546                 }
1547                 r = of_property_read_u32_index(gpio_node, "interrupts", 1, &v1);
1548                 if (r) {
1549                         pr_warn("No \"interrupts\" property.\n");
1550                         return r;
1551                 }
1552                 base_hwirq = (v0 << 6) | v1;
1553         } else {
1554                 pr_warn("Bad \"#interrupt-cells\" property: %u\n",
1555                         interrupt_cells);
1556                 return -EINVAL;
1557         }
1558
1559         gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL);
1560         if (gpiod) {
1561                 /* gpio domain host_data is the base hwirq number. */
1562                 gpiod->base_hwirq = base_hwirq;
1563                 irq_domain_add_linear(
1564                         gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod);
1565         } else {
1566                 pr_warn("Cannot allocate memory for GPIO irq_domain.\n");
1567                 return -ENOMEM;
1568         }
1569
1570         return 0;
1571 }
1572 /*
1573  * Watchdog interrupts are special.  They are associated with a single
1574  * core, so we hardwire the affinity to that core.
1575  */
1576 static void octeon_irq_ciu2_wd_enable(struct irq_data *data)
1577 {
1578         u64 mask;
1579         u64 en_addr;
1580         int coreid = data->irq - OCTEON_IRQ_WDOG0;
1581         struct octeon_ciu_chip_data *cd;
1582
1583         cd = irq_data_get_irq_chip_data(data);
1584         mask = 1ull << (cd->bit);
1585
1586         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1587                 (0x1000ull * cd->line);
1588         cvmx_write_csr(en_addr, mask);
1589
1590 }
1591
1592 static void octeon_irq_ciu2_enable(struct irq_data *data)
1593 {
1594         u64 mask;
1595         u64 en_addr;
1596         int cpu = next_cpu_for_irq(data);
1597         int coreid = octeon_coreid_for_cpu(cpu);
1598         struct octeon_ciu_chip_data *cd;
1599
1600         cd = irq_data_get_irq_chip_data(data);
1601         mask = 1ull << (cd->bit);
1602
1603         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1604                 (0x1000ull * cd->line);
1605         cvmx_write_csr(en_addr, mask);
1606 }
1607
1608 static void octeon_irq_ciu2_enable_local(struct irq_data *data)
1609 {
1610         u64 mask;
1611         u64 en_addr;
1612         int coreid = cvmx_get_core_num();
1613         struct octeon_ciu_chip_data *cd;
1614
1615         cd = irq_data_get_irq_chip_data(data);
1616         mask = 1ull << (cd->bit);
1617
1618         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) +
1619                 (0x1000ull * cd->line);
1620         cvmx_write_csr(en_addr, mask);
1621
1622 }
1623
1624 static void octeon_irq_ciu2_disable_local(struct irq_data *data)
1625 {
1626         u64 mask;
1627         u64 en_addr;
1628         int coreid = cvmx_get_core_num();
1629         struct octeon_ciu_chip_data *cd;
1630
1631         cd = irq_data_get_irq_chip_data(data);
1632         mask = 1ull << (cd->bit);
1633
1634         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) +
1635                 (0x1000ull * cd->line);
1636         cvmx_write_csr(en_addr, mask);
1637
1638 }
1639
1640 static void octeon_irq_ciu2_ack(struct irq_data *data)
1641 {
1642         u64 mask;
1643         u64 en_addr;
1644         int coreid = cvmx_get_core_num();
1645         struct octeon_ciu_chip_data *cd;
1646
1647         cd = irq_data_get_irq_chip_data(data);
1648         mask = 1ull << (cd->bit);
1649
1650         en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd->line);
1651         cvmx_write_csr(en_addr, mask);
1652
1653 }
1654
1655 static void octeon_irq_ciu2_disable_all(struct irq_data *data)
1656 {
1657         int cpu;
1658         u64 mask;
1659         struct octeon_ciu_chip_data *cd;
1660
1661         cd = irq_data_get_irq_chip_data(data);
1662         mask = 1ull << (cd->bit);
1663
1664         for_each_online_cpu(cpu) {
1665                 u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(
1666                         octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd->line);
1667                 cvmx_write_csr(en_addr, mask);
1668         }
1669 }
1670
1671 static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data)
1672 {
1673         int cpu;
1674         u64 mask;
1675
1676         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1677
1678         for_each_online_cpu(cpu) {
1679                 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(
1680                         octeon_coreid_for_cpu(cpu));
1681                 cvmx_write_csr(en_addr, mask);
1682         }
1683 }
1684
1685 static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data)
1686 {
1687         int cpu;
1688         u64 mask;
1689
1690         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1691
1692         for_each_online_cpu(cpu) {
1693                 u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(
1694                         octeon_coreid_for_cpu(cpu));
1695                 cvmx_write_csr(en_addr, mask);
1696         }
1697 }
1698
1699 static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data)
1700 {
1701         u64 mask;
1702         u64 en_addr;
1703         int coreid = cvmx_get_core_num();
1704
1705         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1706         en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid);
1707         cvmx_write_csr(en_addr, mask);
1708 }
1709
1710 static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data)
1711 {
1712         u64 mask;
1713         u64 en_addr;
1714         int coreid = cvmx_get_core_num();
1715
1716         mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1717         en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid);
1718         cvmx_write_csr(en_addr, mask);
1719 }
1720
1721 #ifdef CONFIG_SMP
1722 static int octeon_irq_ciu2_set_affinity(struct irq_data *data,
1723                                         const struct cpumask *dest, bool force)
1724 {
1725         int cpu;
1726         bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
1727         u64 mask;
1728         struct octeon_ciu_chip_data *cd;
1729
1730         if (!enable_one)
1731                 return 0;
1732
1733         cd = irq_data_get_irq_chip_data(data);
1734         mask = 1ull << cd->bit;
1735
1736         for_each_online_cpu(cpu) {
1737                 u64 en_addr;
1738                 if (cpumask_test_cpu(cpu, dest) && enable_one) {
1739                         enable_one = false;
1740                         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(
1741                                 octeon_coreid_for_cpu(cpu)) +
1742                                 (0x1000ull * cd->line);
1743                 } else {
1744                         en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(
1745                                 octeon_coreid_for_cpu(cpu)) +
1746                                 (0x1000ull * cd->line);
1747                 }
1748                 cvmx_write_csr(en_addr, mask);
1749         }
1750
1751         return 0;
1752 }
1753 #endif
1754
1755 static void octeon_irq_ciu2_enable_gpio(struct irq_data *data)
1756 {
1757         octeon_irq_gpio_setup(data);
1758         octeon_irq_ciu2_enable(data);
1759 }
1760
1761 static void octeon_irq_ciu2_disable_gpio(struct irq_data *data)
1762 {
1763         struct octeon_ciu_chip_data *cd;
1764
1765         cd = irq_data_get_irq_chip_data(data);
1766
1767         cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd->gpio_line), 0);
1768
1769         octeon_irq_ciu2_disable_all(data);
1770 }
1771
1772 static struct irq_chip octeon_irq_chip_ciu2 = {
1773         .name = "CIU2-E",
1774         .irq_enable = octeon_irq_ciu2_enable,
1775         .irq_disable = octeon_irq_ciu2_disable_all,
1776         .irq_mask = octeon_irq_ciu2_disable_local,
1777         .irq_unmask = octeon_irq_ciu2_enable,
1778 #ifdef CONFIG_SMP
1779         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1780         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1781 #endif
1782 };
1783
1784 static struct irq_chip octeon_irq_chip_ciu2_edge = {
1785         .name = "CIU2-E",
1786         .irq_enable = octeon_irq_ciu2_enable,
1787         .irq_disable = octeon_irq_ciu2_disable_all,
1788         .irq_ack = octeon_irq_ciu2_ack,
1789         .irq_mask = octeon_irq_ciu2_disable_local,
1790         .irq_unmask = octeon_irq_ciu2_enable,
1791 #ifdef CONFIG_SMP
1792         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1793         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1794 #endif
1795 };
1796
1797 static struct irq_chip octeon_irq_chip_ciu2_mbox = {
1798         .name = "CIU2-M",
1799         .irq_enable = octeon_irq_ciu2_mbox_enable_all,
1800         .irq_disable = octeon_irq_ciu2_mbox_disable_all,
1801         .irq_ack = octeon_irq_ciu2_mbox_disable_local,
1802         .irq_eoi = octeon_irq_ciu2_mbox_enable_local,
1803
1804         .irq_cpu_online = octeon_irq_ciu2_mbox_enable_local,
1805         .irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local,
1806         .flags = IRQCHIP_ONOFFLINE_ENABLED,
1807 };
1808
1809 static struct irq_chip octeon_irq_chip_ciu2_wd = {
1810         .name = "CIU2-W",
1811         .irq_enable = octeon_irq_ciu2_wd_enable,
1812         .irq_disable = octeon_irq_ciu2_disable_all,
1813         .irq_mask = octeon_irq_ciu2_disable_local,
1814         .irq_unmask = octeon_irq_ciu2_enable_local,
1815 };
1816
1817 static struct irq_chip octeon_irq_chip_ciu2_gpio = {
1818         .name = "CIU-GPIO",
1819         .irq_enable = octeon_irq_ciu2_enable_gpio,
1820         .irq_disable = octeon_irq_ciu2_disable_gpio,
1821         .irq_ack = octeon_irq_ciu_gpio_ack,
1822         .irq_mask = octeon_irq_ciu2_disable_local,
1823         .irq_unmask = octeon_irq_ciu2_enable,
1824         .irq_set_type = octeon_irq_ciu_gpio_set_type,
1825 #ifdef CONFIG_SMP
1826         .irq_set_affinity = octeon_irq_ciu2_set_affinity,
1827         .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1828 #endif
1829         .flags = IRQCHIP_SET_TYPE_MASKED,
1830 };
1831
1832 static int octeon_irq_ciu2_xlat(struct irq_domain *d,
1833                                 struct device_node *node,
1834                                 const u32 *intspec,
1835                                 unsigned int intsize,
1836                                 unsigned long *out_hwirq,
1837                                 unsigned int *out_type)
1838 {
1839         unsigned int ciu, bit;
1840
1841         ciu = intspec[0];
1842         bit = intspec[1];
1843
1844         *out_hwirq = (ciu << 6) | bit;
1845         *out_type = 0;
1846
1847         return 0;
1848 }
1849
1850 static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit)
1851 {
1852         bool edge = false;
1853
1854         if (line == 3) /* MIO */
1855                 switch (bit) {
1856                 case 2:  /* IPD_DRP */
1857                 case 8 ... 11: /* Timers */
1858                 case 48: /* PTP */
1859                         edge = true;
1860                         break;
1861                 default:
1862                         break;
1863                 }
1864         else if (line == 6) /* PKT */
1865                 switch (bit) {
1866                 case 52 ... 53: /* ILK_DRP */
1867                 case 8 ... 12:  /* GMX_DRP */
1868                         edge = true;
1869                         break;
1870                 default:
1871                         break;
1872                 }
1873         return edge;
1874 }
1875
1876 static int octeon_irq_ciu2_map(struct irq_domain *d,
1877                                unsigned int virq, irq_hw_number_t hw)
1878 {
1879         unsigned int line = hw >> 6;
1880         unsigned int bit = hw & 63;
1881
1882         /*
1883          * Don't map irq if it is reserved for GPIO.
1884          * (Line 7 are the GPIO lines.)
1885          */
1886         if (line == 7)
1887                 return 0;
1888
1889         if (line > 7 || octeon_irq_ciu_to_irq[line][bit] != 0)
1890                 return -EINVAL;
1891
1892         if (octeon_irq_ciu2_is_edge(line, bit))
1893                 octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1894                                            &octeon_irq_chip_ciu2_edge,
1895                                            handle_edge_irq);
1896         else
1897                 octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1898                                            &octeon_irq_chip_ciu2,
1899                                            handle_level_irq);
1900
1901         return 0;
1902 }
1903
1904 static struct irq_domain_ops octeon_irq_domain_ciu2_ops = {
1905         .map = octeon_irq_ciu2_map,
1906         .unmap = octeon_irq_free_cd,
1907         .xlate = octeon_irq_ciu2_xlat,
1908 };
1909
1910 static void octeon_irq_ciu2(void)
1911 {
1912         int line;
1913         int bit;
1914         int irq;
1915         u64 src_reg, src, sum;
1916         const unsigned long core_id = cvmx_get_core_num();
1917
1918         sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful;
1919
1920         if (unlikely(!sum))
1921                 goto spurious;
1922
1923         line = fls64(sum) - 1;
1924         src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line);
1925         src = cvmx_read_csr(src_reg);
1926
1927         if (unlikely(!src))
1928                 goto spurious;
1929
1930         bit = fls64(src) - 1;
1931         irq = octeon_irq_ciu_to_irq[line][bit];
1932         if (unlikely(!irq))
1933                 goto spurious;
1934
1935         do_IRQ(irq);
1936         goto out;
1937
1938 spurious:
1939         spurious_interrupt();
1940 out:
1941         /* CN68XX pass 1.x has an errata that accessing the ACK registers
1942                 can stop interrupts from propagating */
1943         if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1944                 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1945         else
1946                 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id));
1947         return;
1948 }
1949
1950 static void octeon_irq_ciu2_mbox(void)
1951 {
1952         int line;
1953
1954         const unsigned long core_id = cvmx_get_core_num();
1955         u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60;
1956
1957         if (unlikely(!sum))
1958                 goto spurious;
1959
1960         line = fls64(sum) - 1;
1961
1962         do_IRQ(OCTEON_IRQ_MBOX0 + line);
1963         goto out;
1964
1965 spurious:
1966         spurious_interrupt();
1967 out:
1968         /* CN68XX pass 1.x has an errata that accessing the ACK registers
1969                 can stop interrupts from propagating */
1970         if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1971                 cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1972         else
1973                 cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id));
1974         return;
1975 }
1976
1977 static int __init octeon_irq_init_ciu2(
1978         struct device_node *ciu_node, struct device_node *parent)
1979 {
1980         unsigned int i, r;
1981         struct irq_domain *ciu_domain = NULL;
1982
1983         octeon_irq_init_ciu2_percpu();
1984         octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2;
1985
1986         octeon_irq_gpio_chip = &octeon_irq_chip_ciu2_gpio;
1987         octeon_irq_ip2 = octeon_irq_ciu2;
1988         octeon_irq_ip3 = octeon_irq_ciu2_mbox;
1989         octeon_irq_ip4 = octeon_irq_ip4_mask;
1990
1991         /* Mips internal */
1992         octeon_irq_init_core();
1993
1994         ciu_domain = irq_domain_add_tree(
1995                 ciu_node, &octeon_irq_domain_ciu2_ops, NULL);
1996         irq_set_default_host(ciu_domain);
1997
1998         /* CUI2 */
1999         for (i = 0; i < 64; i++) {
2000                 r = octeon_irq_force_ciu_mapping(
2001                         ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i);
2002                 if (r)
2003                         goto err;
2004         }
2005
2006         for (i = 0; i < 32; i++) {
2007                 r = octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0,
2008                         &octeon_irq_chip_ciu2_wd, handle_level_irq);
2009                 if (r)
2010                         goto err;
2011         }
2012
2013         for (i = 0; i < 4; i++) {
2014                 r = octeon_irq_force_ciu_mapping(
2015                         ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8);
2016                 if (r)
2017                         goto err;
2018         }
2019
2020         r = octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 3, 44);
2021         if (r)
2022                 goto err;
2023
2024         for (i = 0; i < 4; i++) {
2025                 r = octeon_irq_force_ciu_mapping(
2026                         ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i);
2027                 if (r)
2028                         goto err;
2029         }
2030
2031         for (i = 0; i < 4; i++) {
2032                 r = octeon_irq_force_ciu_mapping(
2033                         ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8);
2034                 if (r)
2035                         goto err;
2036         }
2037
2038         irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2039         irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2040         irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2041         irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
2042
2043         /* Enable the CIU lines */
2044         set_c0_status(STATUSF_IP3 | STATUSF_IP2);
2045         clear_c0_status(STATUSF_IP4);
2046         return 0;
2047 err:
2048         return r;
2049 }
2050
2051 struct octeon_irq_cib_host_data {
2052         raw_spinlock_t lock;
2053         u64 raw_reg;
2054         u64 en_reg;
2055         int max_bits;
2056 };
2057
2058 struct octeon_irq_cib_chip_data {
2059         struct octeon_irq_cib_host_data *host_data;
2060         int bit;
2061 };
2062
2063 static void octeon_irq_cib_enable(struct irq_data *data)
2064 {
2065         unsigned long flags;
2066         u64 en;
2067         struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data);
2068         struct octeon_irq_cib_host_data *host_data = cd->host_data;
2069
2070         raw_spin_lock_irqsave(&host_data->lock, flags);
2071         en = cvmx_read_csr(host_data->en_reg);
2072         en |= 1ull << cd->bit;
2073         cvmx_write_csr(host_data->en_reg, en);
2074         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2075 }
2076
2077 static void octeon_irq_cib_disable(struct irq_data *data)
2078 {
2079         unsigned long flags;
2080         u64 en;
2081         struct octeon_irq_cib_chip_data *cd = irq_data_get_irq_chip_data(data);
2082         struct octeon_irq_cib_host_data *host_data = cd->host_data;
2083
2084         raw_spin_lock_irqsave(&host_data->lock, flags);
2085         en = cvmx_read_csr(host_data->en_reg);
2086         en &= ~(1ull << cd->bit);
2087         cvmx_write_csr(host_data->en_reg, en);
2088         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2089 }
2090
2091 static int octeon_irq_cib_set_type(struct irq_data *data, unsigned int t)
2092 {
2093         irqd_set_trigger_type(data, t);
2094         return IRQ_SET_MASK_OK;
2095 }
2096
2097 static struct irq_chip octeon_irq_chip_cib = {
2098         .name = "CIB",
2099         .irq_enable = octeon_irq_cib_enable,
2100         .irq_disable = octeon_irq_cib_disable,
2101         .irq_mask = octeon_irq_cib_disable,
2102         .irq_unmask = octeon_irq_cib_enable,
2103         .irq_set_type = octeon_irq_cib_set_type,
2104 };
2105
2106 static int octeon_irq_cib_xlat(struct irq_domain *d,
2107                                    struct device_node *node,
2108                                    const u32 *intspec,
2109                                    unsigned int intsize,
2110                                    unsigned long *out_hwirq,
2111                                    unsigned int *out_type)
2112 {
2113         unsigned int type = 0;
2114
2115         if (intsize == 2)
2116                 type = intspec[1];
2117
2118         switch (type) {
2119         case 0: /* unofficial value, but we might as well let it work. */
2120         case 4: /* official value for level triggering. */
2121                 *out_type = IRQ_TYPE_LEVEL_HIGH;
2122                 break;
2123         case 1: /* official value for edge triggering. */
2124                 *out_type = IRQ_TYPE_EDGE_RISING;
2125                 break;
2126         default: /* Nothing else is acceptable. */
2127                 return -EINVAL;
2128         }
2129
2130         *out_hwirq = intspec[0];
2131
2132         return 0;
2133 }
2134
2135 static int octeon_irq_cib_map(struct irq_domain *d,
2136                               unsigned int virq, irq_hw_number_t hw)
2137 {
2138         struct octeon_irq_cib_host_data *host_data = d->host_data;
2139         struct octeon_irq_cib_chip_data *cd;
2140
2141         if (hw >= host_data->max_bits) {
2142                 pr_err("ERROR: %s mapping %u is to big!\n",
2143                        irq_domain_get_of_node(d)->name, (unsigned)hw);
2144                 return -EINVAL;
2145         }
2146
2147         cd = kzalloc(sizeof(*cd), GFP_KERNEL);
2148         cd->host_data = host_data;
2149         cd->bit = hw;
2150
2151         irq_set_chip_and_handler(virq, &octeon_irq_chip_cib,
2152                                  handle_simple_irq);
2153         irq_set_chip_data(virq, cd);
2154         return 0;
2155 }
2156
2157 static struct irq_domain_ops octeon_irq_domain_cib_ops = {
2158         .map = octeon_irq_cib_map,
2159         .unmap = octeon_irq_free_cd,
2160         .xlate = octeon_irq_cib_xlat,
2161 };
2162
2163 /* Chain to real handler. */
2164 static irqreturn_t octeon_irq_cib_handler(int my_irq, void *data)
2165 {
2166         u64 en;
2167         u64 raw;
2168         u64 bits;
2169         int i;
2170         int irq;
2171         struct irq_domain *cib_domain = data;
2172         struct octeon_irq_cib_host_data *host_data = cib_domain->host_data;
2173
2174         en = cvmx_read_csr(host_data->en_reg);
2175         raw = cvmx_read_csr(host_data->raw_reg);
2176
2177         bits = en & raw;
2178
2179         for (i = 0; i < host_data->max_bits; i++) {
2180                 if ((bits & 1ull << i) == 0)
2181                         continue;
2182                 irq = irq_find_mapping(cib_domain, i);
2183                 if (!irq) {
2184                         unsigned long flags;
2185
2186                         pr_err("ERROR: CIB bit %d@%llx IRQ unhandled, disabling\n",
2187                                 i, host_data->raw_reg);
2188                         raw_spin_lock_irqsave(&host_data->lock, flags);
2189                         en = cvmx_read_csr(host_data->en_reg);
2190                         en &= ~(1ull << i);
2191                         cvmx_write_csr(host_data->en_reg, en);
2192                         cvmx_write_csr(host_data->raw_reg, 1ull << i);
2193                         raw_spin_unlock_irqrestore(&host_data->lock, flags);
2194                 } else {
2195                         struct irq_desc *desc = irq_to_desc(irq);
2196                         struct irq_data *irq_data = irq_desc_get_irq_data(desc);
2197                         /* If edge, acknowledge the bit we will be sending. */
2198                         if (irqd_get_trigger_type(irq_data) &
2199                                 IRQ_TYPE_EDGE_BOTH)
2200                                 cvmx_write_csr(host_data->raw_reg, 1ull << i);
2201                         generic_handle_irq_desc(desc);
2202                 }
2203         }
2204
2205         return IRQ_HANDLED;
2206 }
2207
2208 static int __init octeon_irq_init_cib(struct device_node *ciu_node,
2209                                       struct device_node *parent)
2210 {
2211         const __be32 *addr;
2212         u32 val;
2213         struct octeon_irq_cib_host_data *host_data;
2214         int parent_irq;
2215         int r;
2216         struct irq_domain *cib_domain;
2217
2218         parent_irq = irq_of_parse_and_map(ciu_node, 0);
2219         if (!parent_irq) {
2220                 pr_err("ERROR: Couldn't acquire parent_irq for %s\n.",
2221                         ciu_node->name);
2222                 return -EINVAL;
2223         }
2224
2225         host_data = kzalloc(sizeof(*host_data), GFP_KERNEL);
2226         raw_spin_lock_init(&host_data->lock);
2227
2228         addr = of_get_address(ciu_node, 0, NULL, NULL);
2229         if (!addr) {
2230                 pr_err("ERROR: Couldn't acquire reg(0) %s\n.", ciu_node->name);
2231                 return -EINVAL;
2232         }
2233         host_data->raw_reg = (u64)phys_to_virt(
2234                 of_translate_address(ciu_node, addr));
2235
2236         addr = of_get_address(ciu_node, 1, NULL, NULL);
2237         if (!addr) {
2238                 pr_err("ERROR: Couldn't acquire reg(1) %s\n.", ciu_node->name);
2239                 return -EINVAL;
2240         }
2241         host_data->en_reg = (u64)phys_to_virt(
2242                 of_translate_address(ciu_node, addr));
2243
2244         r = of_property_read_u32(ciu_node, "cavium,max-bits", &val);
2245         if (r) {
2246                 pr_err("ERROR: Couldn't read cavium,max-bits from %s\n.",
2247                         ciu_node->name);
2248                 return r;
2249         }
2250         host_data->max_bits = val;
2251
2252         cib_domain = irq_domain_add_linear(ciu_node, host_data->max_bits,
2253                                            &octeon_irq_domain_cib_ops,
2254                                            host_data);
2255         if (!cib_domain) {
2256                 pr_err("ERROR: Couldn't irq_domain_add_linear()\n.");
2257                 return -ENOMEM;
2258         }
2259
2260         cvmx_write_csr(host_data->en_reg, 0); /* disable all IRQs */
2261         cvmx_write_csr(host_data->raw_reg, ~0); /* ack any outstanding */
2262
2263         r = request_irq(parent_irq, octeon_irq_cib_handler,
2264                         IRQF_NO_THREAD, "cib", cib_domain);
2265         if (r) {
2266                 pr_err("request_irq cib failed %d\n", r);
2267                 return r;
2268         }
2269         pr_info("CIB interrupt controller probed: %llx %d\n",
2270                 host_data->raw_reg, host_data->max_bits);
2271         return 0;
2272 }
2273
2274 static struct of_device_id ciu_types[] __initdata = {
2275         {.compatible = "cavium,octeon-3860-ciu", .data = octeon_irq_init_ciu},
2276         {.compatible = "cavium,octeon-3860-gpio", .data = octeon_irq_init_gpio},
2277         {.compatible = "cavium,octeon-6880-ciu2", .data = octeon_irq_init_ciu2},
2278         {.compatible = "cavium,octeon-7130-cib", .data = octeon_irq_init_cib},
2279         {}
2280 };
2281
2282 void __init arch_init_irq(void)
2283 {
2284 #ifdef CONFIG_SMP
2285         /* Set the default affinity to the boot cpu. */
2286         cpumask_clear(irq_default_affinity);
2287         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
2288 #endif
2289         of_irq_init(ciu_types);
2290 }
2291
2292 asmlinkage void plat_irq_dispatch(void)
2293 {
2294         unsigned long cop0_cause;
2295         unsigned long cop0_status;
2296
2297         while (1) {
2298                 cop0_cause = read_c0_cause();
2299                 cop0_status = read_c0_status();
2300                 cop0_cause &= cop0_status;
2301                 cop0_cause &= ST0_IM;
2302
2303                 if (cop0_cause & STATUSF_IP2)
2304                         octeon_irq_ip2();
2305                 else if (cop0_cause & STATUSF_IP3)
2306                         octeon_irq_ip3();
2307                 else if (cop0_cause & STATUSF_IP4)
2308                         octeon_irq_ip4();
2309                 else if (cop0_cause)
2310                         do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);
2311                 else
2312                         break;
2313         }
2314 }
2315
2316 #ifdef CONFIG_HOTPLUG_CPU
2317
2318 void octeon_fixup_irqs(void)
2319 {
2320         irq_cpu_offline();
2321 }
2322
2323 #endif /* CONFIG_HOTPLUG_CPU */