kernel/sched/cputime.c

   1 #include <linux/export.h>
   2 #include <linux/sched.h>
   3 #include <linux/tsacct_kern.h>
   4 #include <linux/kernel_stat.h>
   5 #include <linux/static_key.h>
   6 #include <linux/context_tracking.h>
   7 #include "sched.h"
   8 #ifdef CONFIG_PARAVIRT
   9 #include <asm/paravirt.h>
  10 #endif
  11
  12
  13 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
  14
  15 /*
  16  * There are no locks covering percpu hardirq/softirq time.
  17  * They are only modified in vtime_account, on corresponding CPU
  18  * with interrupts disabled. So, writes are safe.
  19  * They are read and saved off onto struct rq in update_rq_clock().
  20  * This may result in other CPU reading this CPU's irq time and can
  21  * race with irq/vtime_account on this CPU. We would either get old
  22  * or new value with a side effect of accounting a slice of irq time to wrong
  23  * task when irq is in progress while we read rq->clock. That is a worthy
  24  * compromise in place of having locks on each irq in account_system_time.
  25  */
  26 DEFINE_PER_CPU(u64, cpu_hardirq_time);
  27 DEFINE_PER_CPU(u64, cpu_softirq_time);
  28
  29 static DEFINE_PER_CPU(u64, irq_start_time);
  30 static int sched_clock_irqtime;
  31
  32 void enable_sched_clock_irqtime(void)
  33 {
  34         sched_clock_irqtime = 1;
  35 }
  36
  37 void disable_sched_clock_irqtime(void)
  38 {
  39         sched_clock_irqtime = 0;
  40 }
  41
  42 #ifndef CONFIG_64BIT
  43 DEFINE_PER_CPU(seqcount_t, irq_time_seq);
  44 #endif /* CONFIG_64BIT */
  45
  46 /*
  47  * Called before incrementing preempt_count on {soft,}irq_enter
  48  * and before decrementing preempt_count on {soft,}irq_exit.
  49  */
  50 void irqtime_account_irq(struct task_struct *curr)
  51 {
  52         unsigned long flags;
  53         s64 delta;
  54         int cpu;
  55
  56         if (!sched_clock_irqtime)
  57                 return;
  58
  59         local_irq_save(flags);
  60
  61         cpu = smp_processor_id();
  62         delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time);
  63         __this_cpu_add(irq_start_time, delta);
  64
  65         irq_time_write_begin();
  66         /*
  67          * We do not account for softirq time from ksoftirqd here.
  68          * We want to continue accounting softirq time to ksoftirqd thread
  69          * in that case, so as not to confuse scheduler with a special task
  70          * that do not consume any time, but still wants to run.
  71          */
  72         if (hardirq_count())
  73                 __this_cpu_add(cpu_hardirq_time, delta);
  74         else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
  75                 __this_cpu_add(cpu_softirq_time, delta);
  76
  77         irq_time_write_end();
  78         local_irq_restore(flags);
  79 }
  80 EXPORT_SYMBOL_GPL(irqtime_account_irq);
  81
  82 static cputime_t irqtime_account_hi_update(cputime_t maxtime)
  83 {
  84         u64 *cpustat = kcpustat_this_cpu->cpustat;
  85         unsigned long flags;
  86         cputime_t irq_cputime;
  87
  88         local_irq_save(flags);
  89         irq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_hardirq_time)) -
  90                       cpustat[CPUTIME_IRQ];
  91         irq_cputime = min(irq_cputime, maxtime);
  92         cpustat[CPUTIME_IRQ] += irq_cputime;
  93         local_irq_restore(flags);
  94         return irq_cputime;
  95 }
  96
  97 static cputime_t irqtime_account_si_update(cputime_t maxtime)
  98 {
  99         u64 *cpustat = kcpustat_this_cpu->cpustat;
 100         unsigned long flags;
 101         cputime_t softirq_cputime;
 102
 103         local_irq_save(flags);
 104         softirq_cputime = nsecs_to_cputime64(this_cpu_read(cpu_softirq_time)) -
 105                           cpustat[CPUTIME_SOFTIRQ];
 106         softirq_cputime = min(softirq_cputime, maxtime);
 107         cpustat[CPUTIME_SOFTIRQ] += softirq_cputime;
 108         local_irq_restore(flags);
 109         return softirq_cputime;
 110 }
 111
 112 #else /* CONFIG_IRQ_TIME_ACCOUNTING */
 113
 114 #define sched_clock_irqtime     (0)
 115
 116 static cputime_t irqtime_account_hi_update(cputime_t dummy)
 117 {
 118         return 0;
 119 }
 120
 121 static cputime_t irqtime_account_si_update(cputime_t dummy)
 122 {
 123         return 0;
 124 }
 125
 126 #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
 127
 128 static inline void task_group_account_field(struct task_struct *p, int index,
 129                                             u64 tmp)
 130 {
 131         /*
 132          * Since all updates are sure to touch the root cgroup, we
 133          * get ourselves ahead and touch it first. If the root cgroup
 134          * is the only cgroup, then nothing else should be necessary.
 135          *
 136          */
 137         __this_cpu_add(kernel_cpustat.cpustat[index], tmp);
 138
 139         cpuacct_account_field(p, index, tmp);
 140 }
 141
 142 /*
 143  * Account user cpu time to a process.
 144  * @p: the process that the cpu time gets accounted to
 145  * @cputime: the cpu time spent in user space since the last update
 146  * @cputime_scaled: cputime scaled by cpu frequency
 147  */
 148 void account_user_time(struct task_struct *p, cputime_t cputime,
 149                        cputime_t cputime_scaled)
 150 {
 151         int index;
 152
 153         /* Add user time to process. */
 154         p->utime += cputime;
 155         p->utimescaled += cputime_scaled;
 156         account_group_user_time(p, cputime);
 157
 158         index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
 159
 160         /* Add user time to cpustat. */
 161         task_group_account_field(p, index, (__force u64) cputime);
 162
 163         /* Account for user time used */
 164         acct_account_cputime(p);
 165 }
 166
 167 /*
 168  * Account guest cpu time to a process.
 169  * @p: the process that the cpu time gets accounted to
 170  * @cputime: the cpu time spent in virtual machine since the last update
 171  * @cputime_scaled: cputime scaled by cpu frequency
 172  */
 173 static void account_guest_time(struct task_struct *p, cputime_t cputime,
 174                                cputime_t cputime_scaled)
 175 {
 176         u64 *cpustat = kcpustat_this_cpu->cpustat;
 177
 178         /* Add guest time to process. */
 179         p->utime += cputime;
 180         p->utimescaled += cputime_scaled;
 181         account_group_user_time(p, cputime);
 182         p->gtime += cputime;
 183
 184         /* Add guest time to cpustat. */
 185         if (task_nice(p) > 0) {
 186                 cpustat[CPUTIME_NICE] += (__force u64) cputime;
 187                 cpustat[CPUTIME_GUEST_NICE] += (__force u64) cputime;
 188         } else {
 189                 cpustat[CPUTIME_USER] += (__force u64) cputime;
 190                 cpustat[CPUTIME_GUEST] += (__force u64) cputime;
 191         }
 192 }
 193
 194 /*
 195  * Account system cpu time to a process and desired cpustat field
 196  * @p: the process that the cpu time gets accounted to
 197  * @cputime: the cpu time spent in kernel space since the last update
 198  * @cputime_scaled: cputime scaled by cpu frequency
 199  * @target_cputime64: pointer to cpustat field that has to be updated
 200  */
 201 static inline
 202 void __account_system_time(struct task_struct *p, cputime_t cputime,
 203                         cputime_t cputime_scaled, int index)
 204 {
 205         /* Add system time to process. */
 206         p->stime += cputime;
 207         p->stimescaled += cputime_scaled;
 208         account_group_system_time(p, cputime);
 209
 210         /* Add system time to cpustat. */
 211         task_group_account_field(p, index, (__force u64) cputime);
 212
 213         /* Account for system time used */
 214         acct_account_cputime(p);
 215 }
 216
 217 /*
 218  * Account system cpu time to a process.
 219  * @p: the process that the cpu time gets accounted to
 220  * @hardirq_offset: the offset to subtract from hardirq_count()
 221  * @cputime: the cpu time spent in kernel space since the last update
 222  * @cputime_scaled: cputime scaled by cpu frequency
 223  */
 224 void account_system_time(struct task_struct *p, int hardirq_offset,
 225                          cputime_t cputime, cputime_t cputime_scaled)
 226 {
 227         int index;
 228
 229         if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
 230                 account_guest_time(p, cputime, cputime_scaled);
 231                 return;
 232         }
 233
 234         if (hardirq_count() - hardirq_offset)
 235                 index = CPUTIME_IRQ;
 236         else if (in_serving_softirq())
 237                 index = CPUTIME_SOFTIRQ;
 238         else
 239                 index = CPUTIME_SYSTEM;
 240
 241         __account_system_time(p, cputime, cputime_scaled, index);
 242 }
 243
 244 /*
 245  * Account for involuntary wait time.
 246  * @cputime: the cpu time spent in involuntary wait
 247  */
 248 void account_steal_time(cputime_t cputime)
 249 {
 250         u64 *cpustat = kcpustat_this_cpu->cpustat;
 251
 252         cpustat[CPUTIME_STEAL] += (__force u64) cputime;
 253 }
 254
 255 /*
 256  * Account for idle time.
 257  * @cputime: the cpu time spent in idle wait
 258  */
 259 void account_idle_time(cputime_t cputime)
 260 {
 261         u64 *cpustat = kcpustat_this_cpu->cpustat;
 262         struct rq *rq = this_rq();
 263
 264         if (atomic_read(&rq->nr_iowait) > 0)
 265                 cpustat[CPUTIME_IOWAIT] += (__force u64) cputime;
 266         else
 267                 cpustat[CPUTIME_IDLE] += (__force u64) cputime;
 268 }
 269
 270 static __always_inline cputime_t steal_account_process_time(cputime_t maxtime)
 271 {
 272 #ifdef CONFIG_PARAVIRT
 273         if (static_key_false(&paravirt_steal_enabled)) {
 274                 cputime_t steal_cputime;
 275                 u64 steal;
 276
 277                 steal = paravirt_steal_clock(smp_processor_id());
 278                 steal -= this_rq()->prev_steal_time;
 279
 280                 steal_cputime = min(nsecs_to_cputime(steal), maxtime);
 281                 account_steal_time(steal_cputime);
 282                 this_rq()->prev_steal_time += cputime_to_nsecs(steal_cputime);
 283
 284                 return steal_cputime;
 285         }
 286 #endif
 287         return 0;
 288 }
 289
 290 /*
 291  * Account how much elapsed time was spent in steal, irq, or softirq time.
 292  */
 293 static inline cputime_t account_other_time(cputime_t max)
 294 {
 295         cputime_t accounted;
 296
 297         accounted = steal_account_process_time(max);
 298
 299         if (accounted < max)
 300                 accounted += irqtime_account_hi_update(max - accounted);
 301
 302         if (accounted < max)
 303                 accounted += irqtime_account_si_update(max - accounted);
 304
 305         return accounted;
 306 }
 307
 308 /*
 309  * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
 310  * tasks (sum on group iteration) belonging to @tsk's group.
 311  */
 312 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
 313 {
 314         struct signal_struct *sig = tsk->signal;
 315         cputime_t utime, stime;
 316         struct task_struct *t;
 317         unsigned int seq, nextseq;
 318         unsigned long flags;
 319
 320         rcu_read_lock();
 321         /* Attempt a lockless read on the first round. */
 322         nextseq = 0;
 323         do {
 324                 seq = nextseq;
 325                 flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);
 326                 times->utime = sig->utime;
 327                 times->stime = sig->stime;
 328                 times->sum_exec_runtime = sig->sum_sched_runtime;
 329
 330                 for_each_thread(tsk, t) {
 331                         task_cputime(t, &utime, &stime);
 332                         times->utime += utime;
 333                         times->stime += stime;
 334                         times->sum_exec_runtime += task_sched_runtime(t);
 335                 }
 336                 /* If lockless access failed, take the lock. */
 337                 nextseq = 1;
 338         } while (need_seqretry(&sig->stats_lock, seq));
 339         done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
 340         rcu_read_unlock();
 341 }
 342
 343 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
 344 /*
 345  * Account a tick to a process and cpustat
 346  * @p: the process that the cpu time gets accounted to
 347  * @user_tick: is the tick from userspace
 348  * @rq: the pointer to rq
 349  *
 350  * Tick demultiplexing follows the order
 351  * - pending hardirq update
 352  * - pending softirq update
 353  * - user_time
 354  * - idle_time
 355  * - system time
 356  *   - check for guest_time
 357  *   - else account as system_time
 358  *
 359  * Check for hardirq is done both for system and user time as there is
 360  * no timer going off while we are on hardirq and hence we may never get an
 361  * opportunity to update it solely in system time.
 362  * p->stime and friends are only updated on system time and not on irq
 363  * softirq as those do not count in task exec_runtime any more.
 364  */
 365 static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
 366                                          struct rq *rq, int ticks)
 367 {
 368         u64 cputime = (__force u64) cputime_one_jiffy * ticks;
 369         cputime_t scaled, other;
 370
 371         /*
 372          * When returning from idle, many ticks can get accounted at
 373          * once, including some ticks of steal, irq, and softirq time.
 374          * Subtract those ticks from the amount of time accounted to
 375          * idle, or potentially user or system time. Due to rounding,
 376          * other time can exceed ticks occasionally.
 377          */
 378         other = account_other_time(cputime);
 379         if (other >= cputime)
 380                 return;
 381         cputime -= other;
 382         scaled = cputime_to_scaled(cputime);
 383
 384         if (this_cpu_ksoftirqd() == p) {
 385                 /*
 386                  * ksoftirqd time do not get accounted in cpu_softirq_time.
 387                  * So, we have to handle it separately here.
 388                  * Also, p->stime needs to be updated for ksoftirqd.
 389                  */
 390                 __account_system_time(p, cputime, scaled, CPUTIME_SOFTIRQ);
 391         } else if (user_tick) {
 392                 account_user_time(p, cputime, scaled);
 393         } else if (p == rq->idle) {
 394                 account_idle_time(cputime);
 395         } else if (p->flags & PF_VCPU) { /* System time or guest time */
 396                 account_guest_time(p, cputime, scaled);
 397         } else {
 398                 __account_system_time(p, cputime, scaled,       CPUTIME_SYSTEM);
 399         }
 400 }
 401
 402 static void irqtime_account_idle_ticks(int ticks)
 403 {
 404         struct rq *rq = this_rq();
 405
 406         irqtime_account_process_tick(current, 0, rq, ticks);
 407 }
 408 #else /* CONFIG_IRQ_TIME_ACCOUNTING */
 409 static inline void irqtime_account_idle_ticks(int ticks) {}
 410 static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
 411                                                 struct rq *rq, int nr_ticks) {}
 412 #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
 413
 414 /*
 415  * Use precise platform statistics if available:
 416  */
 417 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
 418
 419 #ifndef __ARCH_HAS_VTIME_TASK_SWITCH
 420 void vtime_common_task_switch(struct task_struct *prev)
 421 {
 422         if (is_idle_task(prev))
 423                 vtime_account_idle(prev);
 424         else
 425                 vtime_account_system(prev);
 426
 427 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 428         vtime_account_user(prev);
 429 #endif
 430         arch_vtime_task_switch(prev);
 431 }
 432 #endif
 433
 434 /*
 435  * Archs that account the whole time spent in the idle task
 436  * (outside irq) as idle time can rely on this and just implement
 437  * vtime_account_system() and vtime_account_idle(). Archs that
 438  * have other meaning of the idle time (s390 only includes the
 439  * time spent by the CPU when it's in low power mode) must override
 440  * vtime_account().
 441  */
 442 #ifndef __ARCH_HAS_VTIME_ACCOUNT
 443 void vtime_common_account_irq_enter(struct task_struct *tsk)
 444 {
 445         if (!in_interrupt()) {
 446                 /*
 447                  * If we interrupted user, context_tracking_in_user()
 448                  * is 1 because the context tracking don't hook
 449                  * on irq entry/exit. This way we know if
 450                  * we need to flush user time on kernel entry.
 451                  */
 452                 if (context_tracking_in_user()) {
 453                         vtime_account_user(tsk);
 454                         return;
 455                 }
 456
 457                 if (is_idle_task(tsk)) {
 458                         vtime_account_idle(tsk);
 459                         return;
 460                 }
 461         }
 462         vtime_account_system(tsk);
 463 }
 464 EXPORT_SYMBOL_GPL(vtime_common_account_irq_enter);
 465 #endif /* __ARCH_HAS_VTIME_ACCOUNT */
 466 #endif /* CONFIG_VIRT_CPU_ACCOUNTING */
 467
 468
 469 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 470 void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 471 {
 472         *ut = p->utime;
 473         *st = p->stime;
 474 }
 475 EXPORT_SYMBOL_GPL(task_cputime_adjusted);
 476
 477 void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 478 {
 479         struct task_cputime cputime;
 480
 481         thread_group_cputime(p, &cputime);
 482
 483         *ut = cputime.utime;
 484         *st = cputime.stime;
 485 }
 486 #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
 487 /*
 488  * Account a single tick of cpu time.
 489  * @p: the process that the cpu time gets accounted to
 490  * @user_tick: indicates if the tick is a user or a system tick
 491  */
 492 void account_process_tick(struct task_struct *p, int user_tick)
 493 {
 494         cputime_t cputime, scaled, steal;
 495         struct rq *rq = this_rq();
 496
 497         if (vtime_accounting_cpu_enabled())
 498                 return;
 499
 500         if (sched_clock_irqtime) {
 501                 irqtime_account_process_tick(p, user_tick, rq, 1);
 502                 return;
 503         }
 504
 505         cputime = cputime_one_jiffy;
 506         steal = steal_account_process_time(cputime);
 507
 508         if (steal >= cputime)
 509                 return;
 510
 511         cputime -= steal;
 512         scaled = cputime_to_scaled(cputime);
 513
 514         if (user_tick)
 515                 account_user_time(p, cputime, scaled);
 516         else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
 517                 account_system_time(p, HARDIRQ_OFFSET, cputime, scaled);
 518         else
 519                 account_idle_time(cputime);
 520 }
 521
 522 /*
 523  * Account multiple ticks of steal time.
 524  * @p: the process from which the cpu time has been stolen
 525  * @ticks: number of stolen ticks
 526  */
 527 void account_steal_ticks(unsigned long ticks)
 528 {
 529         account_steal_time(jiffies_to_cputime(ticks));
 530 }
 531
 532 /*
 533  * Account multiple ticks of idle time.
 534  * @ticks: number of stolen ticks
 535  */
 536 void account_idle_ticks(unsigned long ticks)
 537 {
 538
 539         if (sched_clock_irqtime) {
 540                 irqtime_account_idle_ticks(ticks);
 541                 return;
 542         }
 543
 544         account_idle_time(jiffies_to_cputime(ticks));
 545 }
 546
 547 /*
 548  * Perform (stime * rtime) / total, but avoid multiplication overflow by
 549  * loosing precision when the numbers are big.
 550  */
 551 static cputime_t scale_stime(u64 stime, u64 rtime, u64 total)
 552 {
 553         u64 scaled;
 554
 555         for (;;) {
 556                 /* Make sure "rtime" is the bigger of stime/rtime */
 557                 if (stime > rtime)
 558                         swap(rtime, stime);
 559
 560                 /* Make sure 'total' fits in 32 bits */
 561                 if (total >> 32)
 562                         goto drop_precision;
 563
 564                 /* Does rtime (and thus stime) fit in 32 bits? */
 565                 if (!(rtime >> 32))
 566                         break;
 567
 568                 /* Can we just balance rtime/stime rather than dropping bits? */
 569                 if (stime >> 31)
 570                         goto drop_precision;
 571
 572                 /* We can grow stime and shrink rtime and try to make them both fit */
 573                 stime <<= 1;
 574                 rtime >>= 1;
 575                 continue;
 576
 577 drop_precision:
 578                 /* We drop from rtime, it has more bits than stime */
 579                 rtime >>= 1;
 580                 total >>= 1;
 581         }
 582
 583         /*
 584          * Make sure gcc understands that this is a 32x32->64 multiply,
 585          * followed by a 64/32->64 divide.
 586          */
 587         scaled = div_u64((u64) (u32) stime * (u64) (u32) rtime, (u32)total);
 588         return (__force cputime_t) scaled;
 589 }
 590
 591 /*
 592  * Adjust tick based cputime random precision against scheduler runtime
 593  * accounting.
 594  *
 595  * Tick based cputime accounting depend on random scheduling timeslices of a
 596  * task to be interrupted or not by the timer.  Depending on these
 597  * circumstances, the number of these interrupts may be over or
 598  * under-optimistic, matching the real user and system cputime with a variable
 599  * precision.
 600  *
 601  * Fix this by scaling these tick based values against the total runtime
 602  * accounted by the CFS scheduler.
 603  *
 604  * This code provides the following guarantees:
 605  *
 606  *   stime + utime == rtime
 607  *   stime_i+1 >= stime_i, utime_i+1 >= utime_i
 608  *
 609  * Assuming that rtime_i+1 >= rtime_i.
 610  */
 611 static void cputime_adjust(struct task_cputime *curr,
 612                            struct prev_cputime *prev,
 613                            cputime_t *ut, cputime_t *st)
 614 {
 615         cputime_t rtime, stime, utime;
 616         unsigned long flags;
 617
 618         /* Serialize concurrent callers such that we can honour our guarantees */
 619         raw_spin_lock_irqsave(&prev->lock, flags);
 620         rtime = nsecs_to_cputime(curr->sum_exec_runtime);
 621
 622         /*
 623          * This is possible under two circumstances:
 624          *  - rtime isn't monotonic after all (a bug);
 625          *  - we got reordered by the lock.
 626          *
 627          * In both cases this acts as a filter such that the rest of the code
 628          * can assume it is monotonic regardless of anything else.
 629          */
 630         if (prev->stime + prev->utime >= rtime)
 631                 goto out;
 632
 633         stime = curr->stime;
 634         utime = curr->utime;
 635
 636         if (utime == 0) {
 637                 stime = rtime;
 638                 goto update;
 639         }
 640
 641         if (stime == 0) {
 642                 utime = rtime;
 643                 goto update;
 644         }
 645
 646         stime = scale_stime((__force u64)stime, (__force u64)rtime,
 647                             (__force u64)(stime + utime));
 648
 649         /*
 650          * Make sure stime doesn't go backwards; this preserves monotonicity
 651          * for utime because rtime is monotonic.
 652          *
 653          *  utime_i+1 = rtime_i+1 - stime_i
 654          *            = rtime_i+1 - (rtime_i - utime_i)
 655          *            = (rtime_i+1 - rtime_i) + utime_i
 656          *            >= utime_i
 657          */
 658         if (stime < prev->stime)
 659                 stime = prev->stime;
 660         utime = rtime - stime;
 661
 662         /*
 663          * Make sure utime doesn't go backwards; this still preserves
 664          * monotonicity for stime, analogous argument to above.
 665          */
 666         if (utime < prev->utime) {
 667                 utime = prev->utime;
 668                 stime = rtime - utime;
 669         }
 670
 671 update:
 672         prev->stime = stime;
 673         prev->utime = utime;
 674 out:
 675         *ut = prev->utime;
 676         *st = prev->stime;
 677         raw_spin_unlock_irqrestore(&prev->lock, flags);
 678 }
 679
 680 void task_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 681 {
 682         struct task_cputime cputime = {
 683                 .sum_exec_runtime = p->se.sum_exec_runtime,
 684         };
 685
 686         task_cputime(p, &cputime.utime, &cputime.stime);
 687         cputime_adjust(&cputime, &p->prev_cputime, ut, st);
 688 }
 689 EXPORT_SYMBOL_GPL(task_cputime_adjusted);
 690
 691 void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, cputime_t *st)
 692 {
 693         struct task_cputime cputime;
 694
 695         thread_group_cputime(p, &cputime);
 696         cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
 697 }
 698 #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
 699
 700 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
 701 static cputime_t vtime_delta(struct task_struct *tsk)
 702 {
 703         unsigned long now = READ_ONCE(jiffies);
 704
 705         if (time_before(now, (unsigned long)tsk->vtime_snap))
 706                 return 0;
 707
 708         return jiffies_to_cputime(now - tsk->vtime_snap);
 709 }
 710
 711 static cputime_t get_vtime_delta(struct task_struct *tsk)
 712 {
 713         unsigned long now = READ_ONCE(jiffies);
 714         cputime_t delta, steal;
 715
 716         delta = jiffies_to_cputime(now - tsk->vtime_snap);
 717         steal = steal_account_process_time(delta);
 718         WARN_ON_ONCE(tsk->vtime_snap_whence == VTIME_INACTIVE);
 719         tsk->vtime_snap = now;
 720
 721         return delta - steal;
 722 }
 723
 724 static void __vtime_account_system(struct task_struct *tsk)
 725 {
 726         cputime_t delta_cpu = get_vtime_delta(tsk);
 727
 728         account_system_time(tsk, irq_count(), delta_cpu, cputime_to_scaled(delta_cpu));
 729 }
 730
 731 void vtime_account_system(struct task_struct *tsk)
 732 {
 733         if (!vtime_delta(tsk))
 734                 return;
 735
 736         write_seqcount_begin(&tsk->vtime_seqcount);
 737         __vtime_account_system(tsk);
 738         write_seqcount_end(&tsk->vtime_seqcount);
 739 }
 740
 741 void vtime_gen_account_irq_exit(struct task_struct *tsk)
 742 {
 743         write_seqcount_begin(&tsk->vtime_seqcount);
 744         if (vtime_delta(tsk))
 745                 __vtime_account_system(tsk);
 746         if (context_tracking_in_user())
 747                 tsk->vtime_snap_whence = VTIME_USER;
 748         write_seqcount_end(&tsk->vtime_seqcount);
 749 }
 750
 751 void vtime_account_user(struct task_struct *tsk)
 752 {
 753         cputime_t delta_cpu;
 754
 755         write_seqcount_begin(&tsk->vtime_seqcount);
 756         tsk->vtime_snap_whence = VTIME_SYS;
 757         if (vtime_delta(tsk)) {
 758                 delta_cpu = get_vtime_delta(tsk);
 759                 account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu));
 760         }
 761         write_seqcount_end(&tsk->vtime_seqcount);
 762 }
 763
 764 void vtime_user_enter(struct task_struct *tsk)
 765 {
 766         write_seqcount_begin(&tsk->vtime_seqcount);
 767         if (vtime_delta(tsk))
 768                 __vtime_account_system(tsk);
 769         tsk->vtime_snap_whence = VTIME_USER;
 770         write_seqcount_end(&tsk->vtime_seqcount);
 771 }
 772
 773 void vtime_guest_enter(struct task_struct *tsk)
 774 {
 775         /*
 776          * The flags must be updated under the lock with
 777          * the vtime_snap flush and update.
 778          * That enforces a right ordering and update sequence
 779          * synchronization against the reader (task_gtime())
 780          * that can thus safely catch up with a tickless delta.
 781          */
 782         write_seqcount_begin(&tsk->vtime_seqcount);
 783         if (vtime_delta(tsk))
 784                 __vtime_account_system(tsk);
 785         current->flags |= PF_VCPU;
 786         write_seqcount_end(&tsk->vtime_seqcount);
 787 }
 788 EXPORT_SYMBOL_GPL(vtime_guest_enter);
 789
 790 void vtime_guest_exit(struct task_struct *tsk)
 791 {
 792         write_seqcount_begin(&tsk->vtime_seqcount);
 793         __vtime_account_system(tsk);
 794         current->flags &= ~PF_VCPU;
 795         write_seqcount_end(&tsk->vtime_seqcount);
 796 }
 797 EXPORT_SYMBOL_GPL(vtime_guest_exit);
 798
 799 void vtime_account_idle(struct task_struct *tsk)
 800 {
 801         cputime_t delta_cpu = get_vtime_delta(tsk);
 802
 803         account_idle_time(delta_cpu);
 804 }
 805
 806 void arch_vtime_task_switch(struct task_struct *prev)
 807 {
 808         write_seqcount_begin(&prev->vtime_seqcount);
 809         prev->vtime_snap_whence = VTIME_INACTIVE;
 810         write_seqcount_end(&prev->vtime_seqcount);
 811
 812         write_seqcount_begin(&current->vtime_seqcount);
 813         current->vtime_snap_whence = VTIME_SYS;
 814         current->vtime_snap = jiffies;
 815         write_seqcount_end(&current->vtime_seqcount);
 816 }
 817
 818 void vtime_init_idle(struct task_struct *t, int cpu)
 819 {
 820         unsigned long flags;
 821
 822         local_irq_save(flags);
 823         write_seqcount_begin(&t->vtime_seqcount);
 824         t->vtime_snap_whence = VTIME_SYS;
 825         t->vtime_snap = jiffies;
 826         write_seqcount_end(&t->vtime_seqcount);
 827         local_irq_restore(flags);
 828 }
 829
 830 cputime_t task_gtime(struct task_struct *t)
 831 {
 832         unsigned int seq;
 833         cputime_t gtime;
 834
 835         if (!vtime_accounting_enabled())
 836                 return t->gtime;
 837
 838         do {
 839                 seq = read_seqcount_begin(&t->vtime_seqcount);
 840
 841                 gtime = t->gtime;
 842                 if (t->vtime_snap_whence == VTIME_SYS && t->flags & PF_VCPU)
 843                         gtime += vtime_delta(t);
 844
 845         } while (read_seqcount_retry(&t->vtime_seqcount, seq));
 846
 847         return gtime;
 848 }
 849
 850 /*
 851  * Fetch cputime raw values from fields of task_struct and
 852  * add up the pending nohz execution time since the last
 853  * cputime snapshot.
 854  */
 855 static void
 856 fetch_task_cputime(struct task_struct *t,
 857                    cputime_t *u_dst, cputime_t *s_dst,
 858                    cputime_t *u_src, cputime_t *s_src,
 859                    cputime_t *udelta, cputime_t *sdelta)
 860 {
 861         unsigned int seq;
 862         unsigned long long delta;
 863
 864         do {
 865                 *udelta = 0;
 866                 *sdelta = 0;
 867
 868                 seq = read_seqcount_begin(&t->vtime_seqcount);
 869
 870                 if (u_dst)
 871                         *u_dst = *u_src;
 872                 if (s_dst)
 873                         *s_dst = *s_src;
 874
 875                 /* Task is sleeping, nothing to add */
 876                 if (t->vtime_snap_whence == VTIME_INACTIVE ||
 877                     is_idle_task(t))
 878                         continue;
 879
 880                 delta = vtime_delta(t);
 881
 882                 /*
 883                  * Task runs either in user or kernel space, add pending nohz time to
 884                  * the right place.
 885                  */
 886                 if (t->vtime_snap_whence == VTIME_USER || t->flags & PF_VCPU) {
 887                         *udelta = delta;
 888                 } else {
 889                         if (t->vtime_snap_whence == VTIME_SYS)
 890                                 *sdelta = delta;
 891                 }
 892         } while (read_seqcount_retry(&t->vtime_seqcount, seq));
 893 }
 894
 895
 896 void task_cputime(struct task_struct *t, cputime_t *utime, cputime_t *stime)
 897 {
 898         cputime_t udelta, sdelta;
 899
 900         if (!vtime_accounting_enabled()) {
 901                 if (utime)
 902                         *utime = t->utime;
 903                 if (stime)
 904                         *stime = t->stime;
 905                 return;
 906         }
 907
 908         fetch_task_cputime(t, utime, stime, &t->utime,
 909                            &t->stime, &udelta, &sdelta);
 910         if (utime)
 911                 *utime += udelta;
 912         if (stime)
 913                 *stime += sdelta;
 914 }
 915
 916 void task_cputime_scaled(struct task_struct *t,
 917                          cputime_t *utimescaled, cputime_t *stimescaled)
 918 {
 919         cputime_t udelta, sdelta;
 920
 921         if (!vtime_accounting_enabled()) {
 922                 if (utimescaled)
 923                         *utimescaled = t->utimescaled;
 924                 if (stimescaled)
 925                         *stimescaled = t->stimescaled;
 926                 return;
 927         }
 928
 929         fetch_task_cputime(t, utimescaled, stimescaled,
 930                            &t->utimescaled, &t->stimescaled, &udelta, &sdelta);
 931         if (utimescaled)
 932                 *utimescaled += cputime_to_scaled(udelta);
 933         if (stimescaled)
 934                 *stimescaled += cputime_to_scaled(sdelta);
 935 }
 936 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */