Version:  2.0.40 2.2.26 2.4.37 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6

Linux/arch/x86/kernel/process_64.c

  1 /*
  2  *  Copyright (C) 1995  Linus Torvalds
  3  *
  4  *  Pentium III FXSR, SSE support
  5  *      Gareth Hughes <gareth@valinux.com>, May 2000
  6  *
  7  *  X86-64 port
  8  *      Andi Kleen.
  9  *
 10  *      CPU hotplug support - ashok.raj@intel.com
 11  */
 12 
 13 /*
 14  * This file handles the architecture-dependent parts of process handling..
 15  */
 16 
 17 #include <linux/cpu.h>
 18 #include <linux/errno.h>
 19 #include <linux/sched.h>
 20 #include <linux/fs.h>
 21 #include <linux/kernel.h>
 22 #include <linux/mm.h>
 23 #include <linux/elfcore.h>
 24 #include <linux/smp.h>
 25 #include <linux/slab.h>
 26 #include <linux/user.h>
 27 #include <linux/interrupt.h>
 28 #include <linux/delay.h>
 29 #include <linux/module.h>
 30 #include <linux/ptrace.h>
 31 #include <linux/notifier.h>
 32 #include <linux/kprobes.h>
 33 #include <linux/kdebug.h>
 34 #include <linux/prctl.h>
 35 #include <linux/uaccess.h>
 36 #include <linux/io.h>
 37 #include <linux/ftrace.h>
 38 
 39 #include <asm/pgtable.h>
 40 #include <asm/processor.h>
 41 #include <asm/fpu/internal.h>
 42 #include <asm/mmu_context.h>
 43 #include <asm/prctl.h>
 44 #include <asm/desc.h>
 45 #include <asm/proto.h>
 46 #include <asm/ia32.h>
 47 #include <asm/idle.h>
 48 #include <asm/syscalls.h>
 49 #include <asm/debugreg.h>
 50 #include <asm/switch_to.h>
 51 #include <asm/xen/hypervisor.h>
 52 
 53 asmlinkage extern void ret_from_fork(void);
 54 
 55 __visible DEFINE_PER_CPU(unsigned long, rsp_scratch);
 56 
 57 /* Prints also some state that isn't saved in the pt_regs */
 58 void __show_regs(struct pt_regs *regs, int all)
 59 {
 60         unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
 61         unsigned long d0, d1, d2, d3, d6, d7;
 62         unsigned int fsindex, gsindex;
 63         unsigned int ds, cs, es;
 64 
 65         printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
 66         printk_address(regs->ip);
 67         printk(KERN_DEFAULT "RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss,
 68                         regs->sp, regs->flags);
 69         printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
 70                regs->ax, regs->bx, regs->cx);
 71         printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
 72                regs->dx, regs->si, regs->di);
 73         printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
 74                regs->bp, regs->r8, regs->r9);
 75         printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
 76                regs->r10, regs->r11, regs->r12);
 77         printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
 78                regs->r13, regs->r14, regs->r15);
 79 
 80         asm("movl %%ds,%0" : "=r" (ds));
 81         asm("movl %%cs,%0" : "=r" (cs));
 82         asm("movl %%es,%0" : "=r" (es));
 83         asm("movl %%fs,%0" : "=r" (fsindex));
 84         asm("movl %%gs,%0" : "=r" (gsindex));
 85 
 86         rdmsrl(MSR_FS_BASE, fs);
 87         rdmsrl(MSR_GS_BASE, gs);
 88         rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
 89 
 90         if (!all)
 91                 return;
 92 
 93         cr0 = read_cr0();
 94         cr2 = read_cr2();
 95         cr3 = read_cr3();
 96         cr4 = __read_cr4();
 97 
 98         printk(KERN_DEFAULT "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
 99                fs, fsindex, gs, gsindex, shadowgs);
100         printk(KERN_DEFAULT "CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
101                         es, cr0);
102         printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
103                         cr4);
104 
105         get_debugreg(d0, 0);
106         get_debugreg(d1, 1);
107         get_debugreg(d2, 2);
108         get_debugreg(d3, 3);
109         get_debugreg(d6, 6);
110         get_debugreg(d7, 7);
111 
112         /* Only print out debug registers if they are in their non-default state. */
113         if ((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
114             (d6 == DR6_RESERVED) && (d7 == 0x400))
115                 return;
116 
117         printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
118         printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
119 
120         if (boot_cpu_has(X86_FEATURE_OSPKE))
121                 printk(KERN_DEFAULT "PKRU: %08x\n", read_pkru());
122 }
123 
124 void release_thread(struct task_struct *dead_task)
125 {
126         if (dead_task->mm) {
127 #ifdef CONFIG_MODIFY_LDT_SYSCALL
128                 if (dead_task->mm->context.ldt) {
129                         pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
130                                 dead_task->comm,
131                                 dead_task->mm->context.ldt->entries,
132                                 dead_task->mm->context.ldt->size);
133                         BUG();
134                 }
135 #endif
136         }
137 }
138 
139 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
140 {
141         struct user_desc ud = {
142                 .base_addr = addr,
143                 .limit = 0xfffff,
144                 .seg_32bit = 1,
145                 .limit_in_pages = 1,
146                 .useable = 1,
147         };
148         struct desc_struct *desc = t->thread.tls_array;
149         desc += tls;
150         fill_ldt(desc, &ud);
151 }
152 
153 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
154 {
155         return get_desc_base(&t->thread.tls_array[tls]);
156 }
157 
158 int copy_thread_tls(unsigned long clone_flags, unsigned long sp,
159                 unsigned long arg, struct task_struct *p, unsigned long tls)
160 {
161         int err;
162         struct pt_regs *childregs;
163         struct task_struct *me = current;
164 
165         p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
166         childregs = task_pt_regs(p);
167         p->thread.sp = (unsigned long) childregs;
168         set_tsk_thread_flag(p, TIF_FORK);
169         p->thread.io_bitmap_ptr = NULL;
170 
171         savesegment(gs, p->thread.gsindex);
172         p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
173         savesegment(fs, p->thread.fsindex);
174         p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
175         savesegment(es, p->thread.es);
176         savesegment(ds, p->thread.ds);
177         memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
178 
179         if (unlikely(p->flags & PF_KTHREAD)) {
180                 /* kernel thread */
181                 memset(childregs, 0, sizeof(struct pt_regs));
182                 childregs->sp = (unsigned long)childregs;
183                 childregs->ss = __KERNEL_DS;
184                 childregs->bx = sp; /* function */
185                 childregs->bp = arg;
186                 childregs->orig_ax = -1;
187                 childregs->cs = __KERNEL_CS | get_kernel_rpl();
188                 childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_FIXED;
189                 return 0;
190         }
191         *childregs = *current_pt_regs();
192 
193         childregs->ax = 0;
194         if (sp)
195                 childregs->sp = sp;
196 
197         err = -ENOMEM;
198         if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
199                 p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
200                                                   IO_BITMAP_BYTES, GFP_KERNEL);
201                 if (!p->thread.io_bitmap_ptr) {
202                         p->thread.io_bitmap_max = 0;
203                         return -ENOMEM;
204                 }
205                 set_tsk_thread_flag(p, TIF_IO_BITMAP);
206         }
207 
208         /*
209          * Set a new TLS for the child thread?
210          */
211         if (clone_flags & CLONE_SETTLS) {
212 #ifdef CONFIG_IA32_EMULATION
213                 if (is_ia32_task())
214                         err = do_set_thread_area(p, -1,
215                                 (struct user_desc __user *)tls, 0);
216                 else
217 #endif
218                         err = do_arch_prctl(p, ARCH_SET_FS, tls);
219                 if (err)
220                         goto out;
221         }
222         err = 0;
223 out:
224         if (err && p->thread.io_bitmap_ptr) {
225                 kfree(p->thread.io_bitmap_ptr);
226                 p->thread.io_bitmap_max = 0;
227         }
228 
229         return err;
230 }
231 
232 static void
233 start_thread_common(struct pt_regs *regs, unsigned long new_ip,
234                     unsigned long new_sp,
235                     unsigned int _cs, unsigned int _ss, unsigned int _ds)
236 {
237         loadsegment(fs, 0);
238         loadsegment(es, _ds);
239         loadsegment(ds, _ds);
240         load_gs_index(0);
241         regs->ip                = new_ip;
242         regs->sp                = new_sp;
243         regs->cs                = _cs;
244         regs->ss                = _ss;
245         regs->flags             = X86_EFLAGS_IF;
246         force_iret();
247 }
248 
249 void
250 start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
251 {
252         start_thread_common(regs, new_ip, new_sp,
253                             __USER_CS, __USER_DS, 0);
254 }
255 
256 #ifdef CONFIG_COMPAT
257 void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp)
258 {
259         start_thread_common(regs, new_ip, new_sp,
260                             test_thread_flag(TIF_X32)
261                             ? __USER_CS : __USER32_CS,
262                             __USER_DS, __USER_DS);
263 }
264 #endif
265 
266 /*
267  *      switch_to(x,y) should switch tasks from x to y.
268  *
269  * This could still be optimized:
270  * - fold all the options into a flag word and test it with a single test.
271  * - could test fs/gs bitsliced
272  *
273  * Kprobes not supported here. Set the probe on schedule instead.
274  * Function graph tracer not supported too.
275  */
276 __visible __notrace_funcgraph struct task_struct *
277 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
278 {
279         struct thread_struct *prev = &prev_p->thread;
280         struct thread_struct *next = &next_p->thread;
281         struct fpu *prev_fpu = &prev->fpu;
282         struct fpu *next_fpu = &next->fpu;
283         int cpu = smp_processor_id();
284         struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
285         unsigned fsindex, gsindex;
286         fpu_switch_t fpu_switch;
287 
288         fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
289 
290         /* We must save %fs and %gs before load_TLS() because
291          * %fs and %gs may be cleared by load_TLS().
292          *
293          * (e.g. xen_load_tls())
294          */
295         savesegment(fs, fsindex);
296         savesegment(gs, gsindex);
297 
298         /*
299          * Load TLS before restoring any segments so that segment loads
300          * reference the correct GDT entries.
301          */
302         load_TLS(next, cpu);
303 
304         /*
305          * Leave lazy mode, flushing any hypercalls made here.  This
306          * must be done after loading TLS entries in the GDT but before
307          * loading segments that might reference them, and and it must
308          * be done before fpu__restore(), so the TS bit is up to
309          * date.
310          */
311         arch_end_context_switch(next_p);
312 
313         /* Switch DS and ES.
314          *
315          * Reading them only returns the selectors, but writing them (if
316          * nonzero) loads the full descriptor from the GDT or LDT.  The
317          * LDT for next is loaded in switch_mm, and the GDT is loaded
318          * above.
319          *
320          * We therefore need to write new values to the segment
321          * registers on every context switch unless both the new and old
322          * values are zero.
323          *
324          * Note that we don't need to do anything for CS and SS, as
325          * those are saved and restored as part of pt_regs.
326          */
327         savesegment(es, prev->es);
328         if (unlikely(next->es | prev->es))
329                 loadsegment(es, next->es);
330 
331         savesegment(ds, prev->ds);
332         if (unlikely(next->ds | prev->ds))
333                 loadsegment(ds, next->ds);
334 
335         /*
336          * Switch FS and GS.
337          *
338          * These are even more complicated than DS and ES: they have
339          * 64-bit bases are that controlled by arch_prctl.  Those bases
340          * only differ from the values in the GDT or LDT if the selector
341          * is 0.
342          *
343          * Loading the segment register resets the hidden base part of
344          * the register to 0 or the value from the GDT / LDT.  If the
345          * next base address zero, writing 0 to the segment register is
346          * much faster than using wrmsr to explicitly zero the base.
347          *
348          * The thread_struct.fs and thread_struct.gs values are 0
349          * if the fs and gs bases respectively are not overridden
350          * from the values implied by fsindex and gsindex.  They
351          * are nonzero, and store the nonzero base addresses, if
352          * the bases are overridden.
353          *
354          * (fs != 0 && fsindex != 0) || (gs != 0 && gsindex != 0) should
355          * be impossible.
356          *
357          * Therefore we need to reload the segment registers if either
358          * the old or new selector is nonzero, and we need to override
359          * the base address if next thread expects it to be overridden.
360          *
361          * This code is unnecessarily slow in the case where the old and
362          * new indexes are zero and the new base is nonzero -- it will
363          * unnecessarily write 0 to the selector before writing the new
364          * base address.
365          *
366          * Note: This all depends on arch_prctl being the only way that
367          * user code can override the segment base.  Once wrfsbase and
368          * wrgsbase are enabled, most of this code will need to change.
369          */
370         if (unlikely(fsindex | next->fsindex | prev->fs)) {
371                 loadsegment(fs, next->fsindex);
372 
373                 /*
374                  * If user code wrote a nonzero value to FS, then it also
375                  * cleared the overridden base address.
376                  *
377                  * XXX: if user code wrote 0 to FS and cleared the base
378                  * address itself, we won't notice and we'll incorrectly
379                  * restore the prior base address next time we reschdule
380                  * the process.
381                  */
382                 if (fsindex)
383                         prev->fs = 0;
384         }
385         if (next->fs)
386                 wrmsrl(MSR_FS_BASE, next->fs);
387         prev->fsindex = fsindex;
388 
389         if (unlikely(gsindex | next->gsindex | prev->gs)) {
390                 load_gs_index(next->gsindex);
391 
392                 /* This works (and fails) the same way as fsindex above. */
393                 if (gsindex)
394                         prev->gs = 0;
395         }
396         if (next->gs)
397                 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
398         prev->gsindex = gsindex;
399 
400         switch_fpu_finish(next_fpu, fpu_switch);
401 
402         /*
403          * Switch the PDA and FPU contexts.
404          */
405         this_cpu_write(current_task, next_p);
406 
407         /* Reload esp0 and ss1.  This changes current_thread_info(). */
408         load_sp0(tss, next);
409 
410         /*
411          * Now maybe reload the debug registers and handle I/O bitmaps
412          */
413         if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
414                      task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
415                 __switch_to_xtra(prev_p, next_p, tss);
416 
417 #ifdef CONFIG_XEN
418         /*
419          * On Xen PV, IOPL bits in pt_regs->flags have no effect, and
420          * current_pt_regs()->flags may not match the current task's
421          * intended IOPL.  We need to switch it manually.
422          */
423         if (unlikely(static_cpu_has(X86_FEATURE_XENPV) &&
424                      prev->iopl != next->iopl))
425                 xen_set_iopl_mask(next->iopl);
426 #endif
427 
428         if (static_cpu_has_bug(X86_BUG_SYSRET_SS_ATTRS)) {
429                 /*
430                  * AMD CPUs have a misfeature: SYSRET sets the SS selector but
431                  * does not update the cached descriptor.  As a result, if we
432                  * do SYSRET while SS is NULL, we'll end up in user mode with
433                  * SS apparently equal to __USER_DS but actually unusable.
434                  *
435                  * The straightforward workaround would be to fix it up just
436                  * before SYSRET, but that would slow down the system call
437                  * fast paths.  Instead, we ensure that SS is never NULL in
438                  * system call context.  We do this by replacing NULL SS
439                  * selectors at every context switch.  SYSCALL sets up a valid
440                  * SS, so the only way to get NULL is to re-enter the kernel
441                  * from CPL 3 through an interrupt.  Since that can't happen
442                  * in the same task as a running syscall, we are guaranteed to
443                  * context switch between every interrupt vector entry and a
444                  * subsequent SYSRET.
445                  *
446                  * We read SS first because SS reads are much faster than
447                  * writes.  Out of caution, we force SS to __KERNEL_DS even if
448                  * it previously had a different non-NULL value.
449                  */
450                 unsigned short ss_sel;
451                 savesegment(ss, ss_sel);
452                 if (ss_sel != __KERNEL_DS)
453                         loadsegment(ss, __KERNEL_DS);
454         }
455 
456         return prev_p;
457 }
458 
459 void set_personality_64bit(void)
460 {
461         /* inherit personality from parent */
462 
463         /* Make sure to be in 64bit mode */
464         clear_thread_flag(TIF_IA32);
465         clear_thread_flag(TIF_ADDR32);
466         clear_thread_flag(TIF_X32);
467 
468         /* Ensure the corresponding mm is not marked. */
469         if (current->mm)
470                 current->mm->context.ia32_compat = 0;
471 
472         /* TBD: overwrites user setup. Should have two bits.
473            But 64bit processes have always behaved this way,
474            so it's not too bad. The main problem is just that
475            32bit childs are affected again. */
476         current->personality &= ~READ_IMPLIES_EXEC;
477 }
478 
479 void set_personality_ia32(bool x32)
480 {
481         /* inherit personality from parent */
482 
483         /* Make sure to be in 32bit mode */
484         set_thread_flag(TIF_ADDR32);
485 
486         /* Mark the associated mm as containing 32-bit tasks. */
487         if (x32) {
488                 clear_thread_flag(TIF_IA32);
489                 set_thread_flag(TIF_X32);
490                 if (current->mm)
491                         current->mm->context.ia32_compat = TIF_X32;
492                 current->personality &= ~READ_IMPLIES_EXEC;
493                 /* in_compat_syscall() uses the presence of the x32
494                    syscall bit flag to determine compat status */
495                 current_thread_info()->status &= ~TS_COMPAT;
496         } else {
497                 set_thread_flag(TIF_IA32);
498                 clear_thread_flag(TIF_X32);
499                 if (current->mm)
500                         current->mm->context.ia32_compat = TIF_IA32;
501                 current->personality |= force_personality32;
502                 /* Prepare the first "return" to user space */
503                 current_thread_info()->status |= TS_COMPAT;
504         }
505 }
506 EXPORT_SYMBOL_GPL(set_personality_ia32);
507 
508 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
509 {
510         int ret = 0;
511         int doit = task == current;
512         int cpu;
513 
514         switch (code) {
515         case ARCH_SET_GS:
516                 if (addr >= TASK_SIZE_OF(task))
517                         return -EPERM;
518                 cpu = get_cpu();
519                 /* handle small bases via the GDT because that's faster to
520                    switch. */
521                 if (addr <= 0xffffffff) {
522                         set_32bit_tls(task, GS_TLS, addr);
523                         if (doit) {
524                                 load_TLS(&task->thread, cpu);
525                                 load_gs_index(GS_TLS_SEL);
526                         }
527                         task->thread.gsindex = GS_TLS_SEL;
528                         task->thread.gs = 0;
529                 } else {
530                         task->thread.gsindex = 0;
531                         task->thread.gs = addr;
532                         if (doit) {
533                                 load_gs_index(0);
534                                 ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr);
535                         }
536                 }
537                 put_cpu();
538                 break;
539         case ARCH_SET_FS:
540                 /* Not strictly needed for fs, but do it for symmetry
541                    with gs */
542                 if (addr >= TASK_SIZE_OF(task))
543                         return -EPERM;
544                 cpu = get_cpu();
545                 /* handle small bases via the GDT because that's faster to
546                    switch. */
547                 if (addr <= 0xffffffff) {
548                         set_32bit_tls(task, FS_TLS, addr);
549                         if (doit) {
550                                 load_TLS(&task->thread, cpu);
551                                 loadsegment(fs, FS_TLS_SEL);
552                         }
553                         task->thread.fsindex = FS_TLS_SEL;
554                         task->thread.fs = 0;
555                 } else {
556                         task->thread.fsindex = 0;
557                         task->thread.fs = addr;
558                         if (doit) {
559                                 /* set the selector to 0 to not confuse
560                                    __switch_to */
561                                 loadsegment(fs, 0);
562                                 ret = wrmsrl_safe(MSR_FS_BASE, addr);
563                         }
564                 }
565                 put_cpu();
566                 break;
567         case ARCH_GET_FS: {
568                 unsigned long base;
569                 if (task->thread.fsindex == FS_TLS_SEL)
570                         base = read_32bit_tls(task, FS_TLS);
571                 else if (doit)
572                         rdmsrl(MSR_FS_BASE, base);
573                 else
574                         base = task->thread.fs;
575                 ret = put_user(base, (unsigned long __user *)addr);
576                 break;
577         }
578         case ARCH_GET_GS: {
579                 unsigned long base;
580                 unsigned gsindex;
581                 if (task->thread.gsindex == GS_TLS_SEL)
582                         base = read_32bit_tls(task, GS_TLS);
583                 else if (doit) {
584                         savesegment(gs, gsindex);
585                         if (gsindex)
586                                 rdmsrl(MSR_KERNEL_GS_BASE, base);
587                         else
588                                 base = task->thread.gs;
589                 } else
590                         base = task->thread.gs;
591                 ret = put_user(base, (unsigned long __user *)addr);
592                 break;
593         }
594 
595         default:
596                 ret = -EINVAL;
597                 break;
598         }
599 
600         return ret;
601 }
602 
603 long sys_arch_prctl(int code, unsigned long addr)
604 {
605         return do_arch_prctl(current, code, addr);
606 }
607 
608 unsigned long KSTK_ESP(struct task_struct *task)
609 {
610         return task_pt_regs(task)->sp;
611 }
612 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us