Version:  2.0.40 2.2.26 2.4.37 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9

Linux/arch/x86/kernel/process.c

  1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  2 
  3 #include <linux/errno.h>
  4 #include <linux/kernel.h>
  5 #include <linux/mm.h>
  6 #include <linux/smp.h>
  7 #include <linux/prctl.h>
  8 #include <linux/slab.h>
  9 #include <linux/sched.h>
 10 #include <linux/init.h>
 11 #include <linux/export.h>
 12 #include <linux/pm.h>
 13 #include <linux/tick.h>
 14 #include <linux/random.h>
 15 #include <linux/user-return-notifier.h>
 16 #include <linux/dmi.h>
 17 #include <linux/utsname.h>
 18 #include <linux/stackprotector.h>
 19 #include <linux/tick.h>
 20 #include <linux/cpuidle.h>
 21 #include <trace/events/power.h>
 22 #include <linux/hw_breakpoint.h>
 23 #include <asm/cpu.h>
 24 #include <asm/apic.h>
 25 #include <asm/syscalls.h>
 26 #include <asm/idle.h>
 27 #include <asm/uaccess.h>
 28 #include <asm/mwait.h>
 29 #include <asm/fpu/internal.h>
 30 #include <asm/debugreg.h>
 31 #include <asm/nmi.h>
 32 #include <asm/tlbflush.h>
 33 #include <asm/mce.h>
 34 #include <asm/vm86.h>
 35 #include <asm/switch_to.h>
 36 
 37 /*
 38  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 39  * no more per-task TSS's. The TSS size is kept cacheline-aligned
 40  * so they are allowed to end up in the .data..cacheline_aligned
 41  * section. Since TSS's are completely CPU-local, we want them
 42  * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 43  */
 44 __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
 45         .x86_tss = {
 46                 .sp0 = TOP_OF_INIT_STACK,
 47 #ifdef CONFIG_X86_32
 48                 .ss0 = __KERNEL_DS,
 49                 .ss1 = __KERNEL_CS,
 50                 .io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
 51 #endif
 52          },
 53 #ifdef CONFIG_X86_32
 54          /*
 55           * Note that the .io_bitmap member must be extra-big. This is because
 56           * the CPU will access an additional byte beyond the end of the IO
 57           * permission bitmap. The extra byte must be all 1 bits, and must
 58           * be within the limit.
 59           */
 60         .io_bitmap              = { [0 ... IO_BITMAP_LONGS] = ~0 },
 61 #endif
 62 #ifdef CONFIG_X86_32
 63         .SYSENTER_stack_canary  = STACK_END_MAGIC,
 64 #endif
 65 };
 66 EXPORT_PER_CPU_SYMBOL(cpu_tss);
 67 
 68 #ifdef CONFIG_X86_64
 69 static DEFINE_PER_CPU(unsigned char, is_idle);
 70 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
 71 
 72 void idle_notifier_register(struct notifier_block *n)
 73 {
 74         atomic_notifier_chain_register(&idle_notifier, n);
 75 }
 76 EXPORT_SYMBOL_GPL(idle_notifier_register);
 77 
 78 void idle_notifier_unregister(struct notifier_block *n)
 79 {
 80         atomic_notifier_chain_unregister(&idle_notifier, n);
 81 }
 82 EXPORT_SYMBOL_GPL(idle_notifier_unregister);
 83 #endif
 84 
 85 /*
 86  * this gets called so that we can store lazy state into memory and copy the
 87  * current task into the new thread.
 88  */
 89 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 90 {
 91         memcpy(dst, src, arch_task_struct_size);
 92 #ifdef CONFIG_VM86
 93         dst->thread.vm86 = NULL;
 94 #endif
 95 
 96         return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
 97 }
 98 
 99 /*
100  * Free current thread data structures etc..
101  */
102 void exit_thread(struct task_struct *tsk)
103 {
104         struct thread_struct *t = &tsk->thread;
105         unsigned long *bp = t->io_bitmap_ptr;
106         struct fpu *fpu = &t->fpu;
107 
108         if (bp) {
109                 struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
110 
111                 t->io_bitmap_ptr = NULL;
112                 clear_thread_flag(TIF_IO_BITMAP);
113                 /*
114                  * Careful, clear this in the TSS too:
115                  */
116                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
117                 t->io_bitmap_max = 0;
118                 put_cpu();
119                 kfree(bp);
120         }
121 
122         free_vm86(t);
123 
124         fpu__drop(fpu);
125 }
126 
127 void flush_thread(void)
128 {
129         struct task_struct *tsk = current;
130 
131         flush_ptrace_hw_breakpoint(tsk);
132         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
133 
134         fpu__clear(&tsk->thread.fpu);
135 }
136 
137 static void hard_disable_TSC(void)
138 {
139         cr4_set_bits(X86_CR4_TSD);
140 }
141 
142 void disable_TSC(void)
143 {
144         preempt_disable();
145         if (!test_and_set_thread_flag(TIF_NOTSC))
146                 /*
147                  * Must flip the CPU state synchronously with
148                  * TIF_NOTSC in the current running context.
149                  */
150                 hard_disable_TSC();
151         preempt_enable();
152 }
153 
154 static void hard_enable_TSC(void)
155 {
156         cr4_clear_bits(X86_CR4_TSD);
157 }
158 
159 static void enable_TSC(void)
160 {
161         preempt_disable();
162         if (test_and_clear_thread_flag(TIF_NOTSC))
163                 /*
164                  * Must flip the CPU state synchronously with
165                  * TIF_NOTSC in the current running context.
166                  */
167                 hard_enable_TSC();
168         preempt_enable();
169 }
170 
171 int get_tsc_mode(unsigned long adr)
172 {
173         unsigned int val;
174 
175         if (test_thread_flag(TIF_NOTSC))
176                 val = PR_TSC_SIGSEGV;
177         else
178                 val = PR_TSC_ENABLE;
179 
180         return put_user(val, (unsigned int __user *)adr);
181 }
182 
183 int set_tsc_mode(unsigned int val)
184 {
185         if (val == PR_TSC_SIGSEGV)
186                 disable_TSC();
187         else if (val == PR_TSC_ENABLE)
188                 enable_TSC();
189         else
190                 return -EINVAL;
191 
192         return 0;
193 }
194 
195 void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
196                       struct tss_struct *tss)
197 {
198         struct thread_struct *prev, *next;
199 
200         prev = &prev_p->thread;
201         next = &next_p->thread;
202 
203         if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
204             test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
205                 unsigned long debugctl = get_debugctlmsr();
206 
207                 debugctl &= ~DEBUGCTLMSR_BTF;
208                 if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
209                         debugctl |= DEBUGCTLMSR_BTF;
210 
211                 update_debugctlmsr(debugctl);
212         }
213 
214         if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
215             test_tsk_thread_flag(next_p, TIF_NOTSC)) {
216                 /* prev and next are different */
217                 if (test_tsk_thread_flag(next_p, TIF_NOTSC))
218                         hard_disable_TSC();
219                 else
220                         hard_enable_TSC();
221         }
222 
223         if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
224                 /*
225                  * Copy the relevant range of the IO bitmap.
226                  * Normally this is 128 bytes or less:
227                  */
228                 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
229                        max(prev->io_bitmap_max, next->io_bitmap_max));
230         } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
231                 /*
232                  * Clear any possible leftover bits:
233                  */
234                 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
235         }
236         propagate_user_return_notify(prev_p, next_p);
237 }
238 
239 /*
240  * Idle related variables and functions
241  */
242 unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
243 EXPORT_SYMBOL(boot_option_idle_override);
244 
245 static void (*x86_idle)(void);
246 
247 #ifndef CONFIG_SMP
248 static inline void play_dead(void)
249 {
250         BUG();
251 }
252 #endif
253 
254 #ifdef CONFIG_X86_64
255 void enter_idle(void)
256 {
257         this_cpu_write(is_idle, 1);
258         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
259 }
260 
261 static void __exit_idle(void)
262 {
263         if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
264                 return;
265         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
266 }
267 
268 /* Called from interrupts to signify idle end */
269 void exit_idle(void)
270 {
271         /* idle loop has pid 0 */
272         if (current->pid)
273                 return;
274         __exit_idle();
275 }
276 #endif
277 
278 void arch_cpu_idle_enter(void)
279 {
280         local_touch_nmi();
281         enter_idle();
282 }
283 
284 void arch_cpu_idle_exit(void)
285 {
286         __exit_idle();
287 }
288 
289 void arch_cpu_idle_dead(void)
290 {
291         play_dead();
292 }
293 
294 /*
295  * Called from the generic idle code.
296  */
297 void arch_cpu_idle(void)
298 {
299         x86_idle();
300 }
301 
302 /*
303  * We use this if we don't have any better idle routine..
304  */
305 void __cpuidle default_idle(void)
306 {
307         trace_cpu_idle_rcuidle(1, smp_processor_id());
308         safe_halt();
309         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
310 }
311 #ifdef CONFIG_APM_MODULE
312 EXPORT_SYMBOL(default_idle);
313 #endif
314 
315 #ifdef CONFIG_XEN
316 bool xen_set_default_idle(void)
317 {
318         bool ret = !!x86_idle;
319 
320         x86_idle = default_idle;
321 
322         return ret;
323 }
324 #endif
325 void stop_this_cpu(void *dummy)
326 {
327         local_irq_disable();
328         /*
329          * Remove this CPU:
330          */
331         set_cpu_online(smp_processor_id(), false);
332         disable_local_APIC();
333         mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
334 
335         for (;;)
336                 halt();
337 }
338 
339 bool amd_e400_c1e_detected;
340 EXPORT_SYMBOL(amd_e400_c1e_detected);
341 
342 static cpumask_var_t amd_e400_c1e_mask;
343 
344 void amd_e400_remove_cpu(int cpu)
345 {
346         if (amd_e400_c1e_mask != NULL)
347                 cpumask_clear_cpu(cpu, amd_e400_c1e_mask);
348 }
349 
350 /*
351  * AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt
352  * pending message MSR. If we detect C1E, then we handle it the same
353  * way as C3 power states (local apic timer and TSC stop)
354  */
355 static void amd_e400_idle(void)
356 {
357         if (!amd_e400_c1e_detected) {
358                 u32 lo, hi;
359 
360                 rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
361 
362                 if (lo & K8_INTP_C1E_ACTIVE_MASK) {
363                         amd_e400_c1e_detected = true;
364                         if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
365                                 mark_tsc_unstable("TSC halt in AMD C1E");
366                         pr_info("System has AMD C1E enabled\n");
367                 }
368         }
369 
370         if (amd_e400_c1e_detected) {
371                 int cpu = smp_processor_id();
372 
373                 if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
374                         cpumask_set_cpu(cpu, amd_e400_c1e_mask);
375                         /* Force broadcast so ACPI can not interfere. */
376                         tick_broadcast_force();
377                         pr_info("Switch to broadcast mode on CPU%d\n", cpu);
378                 }
379                 tick_broadcast_enter();
380 
381                 default_idle();
382 
383                 /*
384                  * The switch back from broadcast mode needs to be
385                  * called with interrupts disabled.
386                  */
387                 local_irq_disable();
388                 tick_broadcast_exit();
389                 local_irq_enable();
390         } else
391                 default_idle();
392 }
393 
394 /*
395  * Intel Core2 and older machines prefer MWAIT over HALT for C1.
396  * We can't rely on cpuidle installing MWAIT, because it will not load
397  * on systems that support only C1 -- so the boot default must be MWAIT.
398  *
399  * Some AMD machines are the opposite, they depend on using HALT.
400  *
401  * So for default C1, which is used during boot until cpuidle loads,
402  * use MWAIT-C1 on Intel HW that has it, else use HALT.
403  */
404 static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
405 {
406         if (c->x86_vendor != X86_VENDOR_INTEL)
407                 return 0;
408 
409         if (!cpu_has(c, X86_FEATURE_MWAIT) || static_cpu_has_bug(X86_BUG_MONITOR))
410                 return 0;
411 
412         return 1;
413 }
414 
415 /*
416  * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
417  * with interrupts enabled and no flags, which is backwards compatible with the
418  * original MWAIT implementation.
419  */
420 static __cpuidle void mwait_idle(void)
421 {
422         if (!current_set_polling_and_test()) {
423                 trace_cpu_idle_rcuidle(1, smp_processor_id());
424                 if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
425                         mb(); /* quirk */
426                         clflush((void *)&current_thread_info()->flags);
427                         mb(); /* quirk */
428                 }
429 
430                 __monitor((void *)&current_thread_info()->flags, 0, 0);
431                 if (!need_resched())
432                         __sti_mwait(0, 0);
433                 else
434                         local_irq_enable();
435                 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
436         } else {
437                 local_irq_enable();
438         }
439         __current_clr_polling();
440 }
441 
442 void select_idle_routine(const struct cpuinfo_x86 *c)
443 {
444 #ifdef CONFIG_SMP
445         if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
446                 pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
447 #endif
448         if (x86_idle || boot_option_idle_override == IDLE_POLL)
449                 return;
450 
451         if (cpu_has_bug(c, X86_BUG_AMD_APIC_C1E)) {
452                 /* E400: APIC timer interrupt does not wake up CPU from C1e */
453                 pr_info("using AMD E400 aware idle routine\n");
454                 x86_idle = amd_e400_idle;
455         } else if (prefer_mwait_c1_over_halt(c)) {
456                 pr_info("using mwait in idle threads\n");
457                 x86_idle = mwait_idle;
458         } else
459                 x86_idle = default_idle;
460 }
461 
462 void __init init_amd_e400_c1e_mask(void)
463 {
464         /* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
465         if (x86_idle == amd_e400_idle)
466                 zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
467 }
468 
469 static int __init idle_setup(char *str)
470 {
471         if (!str)
472                 return -EINVAL;
473 
474         if (!strcmp(str, "poll")) {
475                 pr_info("using polling idle threads\n");
476                 boot_option_idle_override = IDLE_POLL;
477                 cpu_idle_poll_ctrl(true);
478         } else if (!strcmp(str, "halt")) {
479                 /*
480                  * When the boot option of idle=halt is added, halt is
481                  * forced to be used for CPU idle. In such case CPU C2/C3
482                  * won't be used again.
483                  * To continue to load the CPU idle driver, don't touch
484                  * the boot_option_idle_override.
485                  */
486                 x86_idle = default_idle;
487                 boot_option_idle_override = IDLE_HALT;
488         } else if (!strcmp(str, "nomwait")) {
489                 /*
490                  * If the boot option of "idle=nomwait" is added,
491                  * it means that mwait will be disabled for CPU C2/C3
492                  * states. In such case it won't touch the variable
493                  * of boot_option_idle_override.
494                  */
495                 boot_option_idle_override = IDLE_NOMWAIT;
496         } else
497                 return -1;
498 
499         return 0;
500 }
501 early_param("idle", idle_setup);
502 
503 unsigned long arch_align_stack(unsigned long sp)
504 {
505         if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
506                 sp -= get_random_int() % 8192;
507         return sp & ~0xf;
508 }
509 
510 unsigned long arch_randomize_brk(struct mm_struct *mm)
511 {
512         return randomize_page(mm->brk, 0x02000000);
513 }
514 
515 /*
516  * Return saved PC of a blocked thread.
517  * What is this good for? it will be always the scheduler or ret_from_fork.
518  */
519 unsigned long thread_saved_pc(struct task_struct *tsk)
520 {
521         struct inactive_task_frame *frame =
522                 (struct inactive_task_frame *) READ_ONCE(tsk->thread.sp);
523         return READ_ONCE_NOCHECK(frame->ret_addr);
524 }
525 
526 /*
527  * Called from fs/proc with a reference on @p to find the function
528  * which called into schedule(). This needs to be done carefully
529  * because the task might wake up and we might look at a stack
530  * changing under us.
531  */
532 unsigned long get_wchan(struct task_struct *p)
533 {
534         unsigned long start, bottom, top, sp, fp, ip, ret = 0;
535         int count = 0;
536 
537         if (!p || p == current || p->state == TASK_RUNNING)
538                 return 0;
539 
540         if (!try_get_task_stack(p))
541                 return 0;
542 
543         start = (unsigned long)task_stack_page(p);
544         if (!start)
545                 goto out;
546 
547         /*
548          * Layout of the stack page:
549          *
550          * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
551          * PADDING
552          * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
553          * stack
554          * ----------- bottom = start
555          *
556          * The tasks stack pointer points at the location where the
557          * framepointer is stored. The data on the stack is:
558          * ... IP FP ... IP FP
559          *
560          * We need to read FP and IP, so we need to adjust the upper
561          * bound by another unsigned long.
562          */
563         top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
564         top -= 2 * sizeof(unsigned long);
565         bottom = start;
566 
567         sp = READ_ONCE(p->thread.sp);
568         if (sp < bottom || sp > top)
569                 goto out;
570 
571         fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
572         do {
573                 if (fp < bottom || fp > top)
574                         goto out;
575                 ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
576                 if (!in_sched_functions(ip)) {
577                         ret = ip;
578                         goto out;
579                 }
580                 fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
581         } while (count++ < 16 && p->state != TASK_RUNNING);
582 
583 out:
584         put_task_stack(p);
585         return ret;
586 }
587 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us