Version:  2.0.40 2.2.26 2.4.37 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1

Linux/arch/x86/kernel/process.c

  1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  2 
  3 #include <linux/errno.h>
  4 #include <linux/kernel.h>
  5 #include <linux/mm.h>
  6 #include <linux/smp.h>
  7 #include <linux/prctl.h>
  8 #include <linux/slab.h>
  9 #include <linux/sched.h>
 10 #include <linux/module.h>
 11 #include <linux/pm.h>
 12 #include <linux/tick.h>
 13 #include <linux/random.h>
 14 #include <linux/user-return-notifier.h>
 15 #include <linux/dmi.h>
 16 #include <linux/utsname.h>
 17 #include <linux/stackprotector.h>
 18 #include <linux/tick.h>
 19 #include <linux/cpuidle.h>
 20 #include <trace/events/power.h>
 21 #include <linux/hw_breakpoint.h>
 22 #include <asm/cpu.h>
 23 #include <asm/apic.h>
 24 #include <asm/syscalls.h>
 25 #include <asm/idle.h>
 26 #include <asm/uaccess.h>
 27 #include <asm/mwait.h>
 28 #include <asm/i387.h>
 29 #include <asm/fpu-internal.h>
 30 #include <asm/debugreg.h>
 31 #include <asm/nmi.h>
 32 #include <asm/tlbflush.h>
 33 
 34 /*
 35  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 36  * no more per-task TSS's. The TSS size is kept cacheline-aligned
 37  * so they are allowed to end up in the .data..cacheline_aligned
 38  * section. Since TSS's are completely CPU-local, we want them
 39  * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 40  */
 41 __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
 42         .x86_tss = {
 43                 .sp0 = TOP_OF_INIT_STACK,
 44 #ifdef CONFIG_X86_32
 45                 .ss0 = __KERNEL_DS,
 46                 .ss1 = __KERNEL_CS,
 47                 .io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
 48 #endif
 49          },
 50 #ifdef CONFIG_X86_32
 51          /*
 52           * Note that the .io_bitmap member must be extra-big. This is because
 53           * the CPU will access an additional byte beyond the end of the IO
 54           * permission bitmap. The extra byte must be all 1 bits, and must
 55           * be within the limit.
 56           */
 57         .io_bitmap              = { [0 ... IO_BITMAP_LONGS] = ~0 },
 58 #endif
 59 };
 60 EXPORT_PER_CPU_SYMBOL(cpu_tss);
 61 
 62 #ifdef CONFIG_X86_64
 63 static DEFINE_PER_CPU(unsigned char, is_idle);
 64 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
 65 
 66 void idle_notifier_register(struct notifier_block *n)
 67 {
 68         atomic_notifier_chain_register(&idle_notifier, n);
 69 }
 70 EXPORT_SYMBOL_GPL(idle_notifier_register);
 71 
 72 void idle_notifier_unregister(struct notifier_block *n)
 73 {
 74         atomic_notifier_chain_unregister(&idle_notifier, n);
 75 }
 76 EXPORT_SYMBOL_GPL(idle_notifier_unregister);
 77 #endif
 78 
 79 struct kmem_cache *task_xstate_cachep;
 80 EXPORT_SYMBOL_GPL(task_xstate_cachep);
 81 
 82 /*
 83  * this gets called so that we can store lazy state into memory and copy the
 84  * current task into the new thread.
 85  */
 86 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 87 {
 88         *dst = *src;
 89 
 90         dst->thread.fpu_counter = 0;
 91         dst->thread.fpu.has_fpu = 0;
 92         dst->thread.fpu.state = NULL;
 93         task_disable_lazy_fpu_restore(dst);
 94         if (tsk_used_math(src)) {
 95                 int err = fpu_alloc(&dst->thread.fpu);
 96                 if (err)
 97                         return err;
 98                 fpu_copy(dst, src);
 99         }
100         return 0;
101 }
102 
103 void free_thread_xstate(struct task_struct *tsk)
104 {
105         fpu_free(&tsk->thread.fpu);
106 }
107 
108 void arch_release_task_struct(struct task_struct *tsk)
109 {
110         free_thread_xstate(tsk);
111 }
112 
113 void arch_task_cache_init(void)
114 {
115         task_xstate_cachep =
116                 kmem_cache_create("task_xstate", xstate_size,
117                                   __alignof__(union thread_xstate),
118                                   SLAB_PANIC | SLAB_NOTRACK, NULL);
119         setup_xstate_comp();
120 }
121 
122 /*
123  * Free current thread data structures etc..
124  */
125 void exit_thread(void)
126 {
127         struct task_struct *me = current;
128         struct thread_struct *t = &me->thread;
129         unsigned long *bp = t->io_bitmap_ptr;
130 
131         if (bp) {
132                 struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu());
133 
134                 t->io_bitmap_ptr = NULL;
135                 clear_thread_flag(TIF_IO_BITMAP);
136                 /*
137                  * Careful, clear this in the TSS too:
138                  */
139                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
140                 t->io_bitmap_max = 0;
141                 put_cpu();
142                 kfree(bp);
143         }
144 
145         drop_fpu(me);
146 }
147 
148 void flush_thread(void)
149 {
150         struct task_struct *tsk = current;
151 
152         flush_ptrace_hw_breakpoint(tsk);
153         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
154 
155         if (!use_eager_fpu()) {
156                 /* FPU state will be reallocated lazily at the first use. */
157                 drop_fpu(tsk);
158                 free_thread_xstate(tsk);
159         } else {
160                 if (!tsk_used_math(tsk)) {
161                         /* kthread execs. TODO: cleanup this horror. */
162                         if (WARN_ON(init_fpu(tsk)))
163                                 force_sig(SIGKILL, tsk);
164                         user_fpu_begin();
165                 }
166                 restore_init_xstate();
167         }
168 }
169 
170 static void hard_disable_TSC(void)
171 {
172         cr4_set_bits(X86_CR4_TSD);
173 }
174 
175 void disable_TSC(void)
176 {
177         preempt_disable();
178         if (!test_and_set_thread_flag(TIF_NOTSC))
179                 /*
180                  * Must flip the CPU state synchronously with
181                  * TIF_NOTSC in the current running context.
182                  */
183                 hard_disable_TSC();
184         preempt_enable();
185 }
186 
187 static void hard_enable_TSC(void)
188 {
189         cr4_clear_bits(X86_CR4_TSD);
190 }
191 
192 static void enable_TSC(void)
193 {
194         preempt_disable();
195         if (test_and_clear_thread_flag(TIF_NOTSC))
196                 /*
197                  * Must flip the CPU state synchronously with
198                  * TIF_NOTSC in the current running context.
199                  */
200                 hard_enable_TSC();
201         preempt_enable();
202 }
203 
204 int get_tsc_mode(unsigned long adr)
205 {
206         unsigned int val;
207 
208         if (test_thread_flag(TIF_NOTSC))
209                 val = PR_TSC_SIGSEGV;
210         else
211                 val = PR_TSC_ENABLE;
212 
213         return put_user(val, (unsigned int __user *)adr);
214 }
215 
216 int set_tsc_mode(unsigned int val)
217 {
218         if (val == PR_TSC_SIGSEGV)
219                 disable_TSC();
220         else if (val == PR_TSC_ENABLE)
221                 enable_TSC();
222         else
223                 return -EINVAL;
224 
225         return 0;
226 }
227 
228 void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
229                       struct tss_struct *tss)
230 {
231         struct thread_struct *prev, *next;
232 
233         prev = &prev_p->thread;
234         next = &next_p->thread;
235 
236         if (test_tsk_thread_flag(prev_p, TIF_BLOCKSTEP) ^
237             test_tsk_thread_flag(next_p, TIF_BLOCKSTEP)) {
238                 unsigned long debugctl = get_debugctlmsr();
239 
240                 debugctl &= ~DEBUGCTLMSR_BTF;
241                 if (test_tsk_thread_flag(next_p, TIF_BLOCKSTEP))
242                         debugctl |= DEBUGCTLMSR_BTF;
243 
244                 update_debugctlmsr(debugctl);
245         }
246 
247         if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
248             test_tsk_thread_flag(next_p, TIF_NOTSC)) {
249                 /* prev and next are different */
250                 if (test_tsk_thread_flag(next_p, TIF_NOTSC))
251                         hard_disable_TSC();
252                 else
253                         hard_enable_TSC();
254         }
255 
256         if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
257                 /*
258                  * Copy the relevant range of the IO bitmap.
259                  * Normally this is 128 bytes or less:
260                  */
261                 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
262                        max(prev->io_bitmap_max, next->io_bitmap_max));
263         } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
264                 /*
265                  * Clear any possible leftover bits:
266                  */
267                 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
268         }
269         propagate_user_return_notify(prev_p, next_p);
270 }
271 
272 /*
273  * Idle related variables and functions
274  */
275 unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
276 EXPORT_SYMBOL(boot_option_idle_override);
277 
278 static void (*x86_idle)(void);
279 
280 #ifndef CONFIG_SMP
281 static inline void play_dead(void)
282 {
283         BUG();
284 }
285 #endif
286 
287 #ifdef CONFIG_X86_64
288 void enter_idle(void)
289 {
290         this_cpu_write(is_idle, 1);
291         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
292 }
293 
294 static void __exit_idle(void)
295 {
296         if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
297                 return;
298         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
299 }
300 
301 /* Called from interrupts to signify idle end */
302 void exit_idle(void)
303 {
304         /* idle loop has pid 0 */
305         if (current->pid)
306                 return;
307         __exit_idle();
308 }
309 #endif
310 
311 void arch_cpu_idle_enter(void)
312 {
313         local_touch_nmi();
314         enter_idle();
315 }
316 
317 void arch_cpu_idle_exit(void)
318 {
319         __exit_idle();
320 }
321 
322 void arch_cpu_idle_dead(void)
323 {
324         play_dead();
325 }
326 
327 /*
328  * Called from the generic idle code.
329  */
330 void arch_cpu_idle(void)
331 {
332         x86_idle();
333 }
334 
335 /*
336  * We use this if we don't have any better idle routine..
337  */
338 void default_idle(void)
339 {
340         trace_cpu_idle_rcuidle(1, smp_processor_id());
341         safe_halt();
342         trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
343 }
344 #ifdef CONFIG_APM_MODULE
345 EXPORT_SYMBOL(default_idle);
346 #endif
347 
348 #ifdef CONFIG_XEN
349 bool xen_set_default_idle(void)
350 {
351         bool ret = !!x86_idle;
352 
353         x86_idle = default_idle;
354 
355         return ret;
356 }
357 #endif
358 void stop_this_cpu(void *dummy)
359 {
360         local_irq_disable();
361         /*
362          * Remove this CPU:
363          */
364         set_cpu_online(smp_processor_id(), false);
365         disable_local_APIC();
366 
367         for (;;)
368                 halt();
369 }
370 
371 bool amd_e400_c1e_detected;
372 EXPORT_SYMBOL(amd_e400_c1e_detected);
373 
374 static cpumask_var_t amd_e400_c1e_mask;
375 
376 void amd_e400_remove_cpu(int cpu)
377 {
378         if (amd_e400_c1e_mask != NULL)
379                 cpumask_clear_cpu(cpu, amd_e400_c1e_mask);
380 }
381 
382 /*
383  * AMD Erratum 400 aware idle routine. We check for C1E active in the interrupt
384  * pending message MSR. If we detect C1E, then we handle it the same
385  * way as C3 power states (local apic timer and TSC stop)
386  */
387 static void amd_e400_idle(void)
388 {
389         if (!amd_e400_c1e_detected) {
390                 u32 lo, hi;
391 
392                 rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
393 
394                 if (lo & K8_INTP_C1E_ACTIVE_MASK) {
395                         amd_e400_c1e_detected = true;
396                         if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
397                                 mark_tsc_unstable("TSC halt in AMD C1E");
398                         pr_info("System has AMD C1E enabled\n");
399                 }
400         }
401 
402         if (amd_e400_c1e_detected) {
403                 int cpu = smp_processor_id();
404 
405                 if (!cpumask_test_cpu(cpu, amd_e400_c1e_mask)) {
406                         cpumask_set_cpu(cpu, amd_e400_c1e_mask);
407                         /* Force broadcast so ACPI can not interfere. */
408                         tick_broadcast_force();
409                         pr_info("Switch to broadcast mode on CPU%d\n", cpu);
410                 }
411                 tick_broadcast_enter();
412 
413                 default_idle();
414 
415                 /*
416                  * The switch back from broadcast mode needs to be
417                  * called with interrupts disabled.
418                  */
419                 local_irq_disable();
420                 tick_broadcast_exit();
421                 local_irq_enable();
422         } else
423                 default_idle();
424 }
425 
426 /*
427  * Intel Core2 and older machines prefer MWAIT over HALT for C1.
428  * We can't rely on cpuidle installing MWAIT, because it will not load
429  * on systems that support only C1 -- so the boot default must be MWAIT.
430  *
431  * Some AMD machines are the opposite, they depend on using HALT.
432  *
433  * So for default C1, which is used during boot until cpuidle loads,
434  * use MWAIT-C1 on Intel HW that has it, else use HALT.
435  */
436 static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
437 {
438         if (c->x86_vendor != X86_VENDOR_INTEL)
439                 return 0;
440 
441         if (!cpu_has(c, X86_FEATURE_MWAIT))
442                 return 0;
443 
444         return 1;
445 }
446 
447 /*
448  * MONITOR/MWAIT with no hints, used for default default C1 state.
449  * This invokes MWAIT with interrutps enabled and no flags,
450  * which is backwards compatible with the original MWAIT implementation.
451  */
452 
453 static void mwait_idle(void)
454 {
455         if (!current_set_polling_and_test()) {
456                 if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
457                         smp_mb(); /* quirk */
458                         clflush((void *)&current_thread_info()->flags);
459                         smp_mb(); /* quirk */
460                 }
461 
462                 __monitor((void *)&current_thread_info()->flags, 0, 0);
463                 if (!need_resched())
464                         __sti_mwait(0, 0);
465                 else
466                         local_irq_enable();
467         } else {
468                 local_irq_enable();
469         }
470         __current_clr_polling();
471 }
472 
473 void select_idle_routine(const struct cpuinfo_x86 *c)
474 {
475 #ifdef CONFIG_SMP
476         if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
477                 pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
478 #endif
479         if (x86_idle || boot_option_idle_override == IDLE_POLL)
480                 return;
481 
482         if (cpu_has_bug(c, X86_BUG_AMD_APIC_C1E)) {
483                 /* E400: APIC timer interrupt does not wake up CPU from C1e */
484                 pr_info("using AMD E400 aware idle routine\n");
485                 x86_idle = amd_e400_idle;
486         } else if (prefer_mwait_c1_over_halt(c)) {
487                 pr_info("using mwait in idle threads\n");
488                 x86_idle = mwait_idle;
489         } else
490                 x86_idle = default_idle;
491 }
492 
493 void __init init_amd_e400_c1e_mask(void)
494 {
495         /* If we're using amd_e400_idle, we need to allocate amd_e400_c1e_mask. */
496         if (x86_idle == amd_e400_idle)
497                 zalloc_cpumask_var(&amd_e400_c1e_mask, GFP_KERNEL);
498 }
499 
500 static int __init idle_setup(char *str)
501 {
502         if (!str)
503                 return -EINVAL;
504 
505         if (!strcmp(str, "poll")) {
506                 pr_info("using polling idle threads\n");
507                 boot_option_idle_override = IDLE_POLL;
508                 cpu_idle_poll_ctrl(true);
509         } else if (!strcmp(str, "halt")) {
510                 /*
511                  * When the boot option of idle=halt is added, halt is
512                  * forced to be used for CPU idle. In such case CPU C2/C3
513                  * won't be used again.
514                  * To continue to load the CPU idle driver, don't touch
515                  * the boot_option_idle_override.
516                  */
517                 x86_idle = default_idle;
518                 boot_option_idle_override = IDLE_HALT;
519         } else if (!strcmp(str, "nomwait")) {
520                 /*
521                  * If the boot option of "idle=nomwait" is added,
522                  * it means that mwait will be disabled for CPU C2/C3
523                  * states. In such case it won't touch the variable
524                  * of boot_option_idle_override.
525                  */
526                 boot_option_idle_override = IDLE_NOMWAIT;
527         } else
528                 return -1;
529 
530         return 0;
531 }
532 early_param("idle", idle_setup);
533 
534 unsigned long arch_align_stack(unsigned long sp)
535 {
536         if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
537                 sp -= get_random_int() % 8192;
538         return sp & ~0xf;
539 }
540 
541 unsigned long arch_randomize_brk(struct mm_struct *mm)
542 {
543         unsigned long range_end = mm->brk + 0x02000000;
544         return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
545 }
546 
547 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us