Version:  2.0.40 2.2.26 2.4.37 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19

Linux/arch/x86/kernel/process_64.c

  1 /*
  2  *  Copyright (C) 1995  Linus Torvalds
  3  *
  4  *  Pentium III FXSR, SSE support
  5  *      Gareth Hughes <gareth@valinux.com>, May 2000
  6  *
  7  *  X86-64 port
  8  *      Andi Kleen.
  9  *
 10  *      CPU hotplug support - ashok.raj@intel.com
 11  */
 12 
 13 /*
 14  * This file handles the architecture-dependent parts of process handling..
 15  */
 16 
 17 #include <linux/cpu.h>
 18 #include <linux/errno.h>
 19 #include <linux/sched.h>
 20 #include <linux/fs.h>
 21 #include <linux/kernel.h>
 22 #include <linux/mm.h>
 23 #include <linux/elfcore.h>
 24 #include <linux/smp.h>
 25 #include <linux/slab.h>
 26 #include <linux/user.h>
 27 #include <linux/interrupt.h>
 28 #include <linux/delay.h>
 29 #include <linux/module.h>
 30 #include <linux/ptrace.h>
 31 #include <linux/notifier.h>
 32 #include <linux/kprobes.h>
 33 #include <linux/kdebug.h>
 34 #include <linux/prctl.h>
 35 #include <linux/uaccess.h>
 36 #include <linux/io.h>
 37 #include <linux/ftrace.h>
 38 
 39 #include <asm/pgtable.h>
 40 #include <asm/processor.h>
 41 #include <asm/i387.h>
 42 #include <asm/fpu-internal.h>
 43 #include <asm/mmu_context.h>
 44 #include <asm/prctl.h>
 45 #include <asm/desc.h>
 46 #include <asm/proto.h>
 47 #include <asm/ia32.h>
 48 #include <asm/idle.h>
 49 #include <asm/syscalls.h>
 50 #include <asm/debugreg.h>
 51 #include <asm/switch_to.h>
 52 
 53 asmlinkage extern void ret_from_fork(void);
 54 
 55 __visible DEFINE_PER_CPU(unsigned long, old_rsp);
 56 
 57 /* Prints also some state that isn't saved in the pt_regs */
 58 void __show_regs(struct pt_regs *regs, int all)
 59 {
 60         unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
 61         unsigned long d0, d1, d2, d3, d6, d7;
 62         unsigned int fsindex, gsindex;
 63         unsigned int ds, cs, es;
 64 
 65         printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
 66         printk_address(regs->ip);
 67         printk(KERN_DEFAULT "RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss,
 68                         regs->sp, regs->flags);
 69         printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
 70                regs->ax, regs->bx, regs->cx);
 71         printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
 72                regs->dx, regs->si, regs->di);
 73         printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
 74                regs->bp, regs->r8, regs->r9);
 75         printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
 76                regs->r10, regs->r11, regs->r12);
 77         printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
 78                regs->r13, regs->r14, regs->r15);
 79 
 80         asm("movl %%ds,%0" : "=r" (ds));
 81         asm("movl %%cs,%0" : "=r" (cs));
 82         asm("movl %%es,%0" : "=r" (es));
 83         asm("movl %%fs,%0" : "=r" (fsindex));
 84         asm("movl %%gs,%0" : "=r" (gsindex));
 85 
 86         rdmsrl(MSR_FS_BASE, fs);
 87         rdmsrl(MSR_GS_BASE, gs);
 88         rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
 89 
 90         if (!all)
 91                 return;
 92 
 93         cr0 = read_cr0();
 94         cr2 = read_cr2();
 95         cr3 = read_cr3();
 96         cr4 = read_cr4();
 97 
 98         printk(KERN_DEFAULT "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
 99                fs, fsindex, gs, gsindex, shadowgs);
100         printk(KERN_DEFAULT "CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
101                         es, cr0);
102         printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
103                         cr4);
104 
105         get_debugreg(d0, 0);
106         get_debugreg(d1, 1);
107         get_debugreg(d2, 2);
108         get_debugreg(d3, 3);
109         get_debugreg(d6, 6);
110         get_debugreg(d7, 7);
111 
112         /* Only print out debug registers if they are in their non-default state. */
113         if ((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&
114             (d6 == DR6_RESERVED) && (d7 == 0x400))
115                 return;
116 
117         printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
118         printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
119 
120 }
121 
122 void release_thread(struct task_struct *dead_task)
123 {
124         if (dead_task->mm) {
125                 if (dead_task->mm->context.size) {
126                         pr_warn("WARNING: dead process %s still has LDT? <%p/%d>\n",
127                                 dead_task->comm,
128                                 dead_task->mm->context.ldt,
129                                 dead_task->mm->context.size);
130                         BUG();
131                 }
132         }
133 }
134 
135 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
136 {
137         struct user_desc ud = {
138                 .base_addr = addr,
139                 .limit = 0xfffff,
140                 .seg_32bit = 1,
141                 .limit_in_pages = 1,
142                 .useable = 1,
143         };
144         struct desc_struct *desc = t->thread.tls_array;
145         desc += tls;
146         fill_ldt(desc, &ud);
147 }
148 
149 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
150 {
151         return get_desc_base(&t->thread.tls_array[tls]);
152 }
153 
154 int copy_thread(unsigned long clone_flags, unsigned long sp,
155                 unsigned long arg, struct task_struct *p)
156 {
157         int err;
158         struct pt_regs *childregs;
159         struct task_struct *me = current;
160 
161         p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
162         childregs = task_pt_regs(p);
163         p->thread.sp = (unsigned long) childregs;
164         p->thread.usersp = me->thread.usersp;
165         set_tsk_thread_flag(p, TIF_FORK);
166         p->thread.io_bitmap_ptr = NULL;
167 
168         savesegment(gs, p->thread.gsindex);
169         p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
170         savesegment(fs, p->thread.fsindex);
171         p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
172         savesegment(es, p->thread.es);
173         savesegment(ds, p->thread.ds);
174         memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
175 
176         if (unlikely(p->flags & PF_KTHREAD)) {
177                 /* kernel thread */
178                 memset(childregs, 0, sizeof(struct pt_regs));
179                 childregs->sp = (unsigned long)childregs;
180                 childregs->ss = __KERNEL_DS;
181                 childregs->bx = sp; /* function */
182                 childregs->bp = arg;
183                 childregs->orig_ax = -1;
184                 childregs->cs = __KERNEL_CS | get_kernel_rpl();
185                 childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_FIXED;
186                 return 0;
187         }
188         *childregs = *current_pt_regs();
189 
190         childregs->ax = 0;
191         if (sp)
192                 childregs->sp = sp;
193 
194         err = -ENOMEM;
195         if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
196                 p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
197                                                   IO_BITMAP_BYTES, GFP_KERNEL);
198                 if (!p->thread.io_bitmap_ptr) {
199                         p->thread.io_bitmap_max = 0;
200                         return -ENOMEM;
201                 }
202                 set_tsk_thread_flag(p, TIF_IO_BITMAP);
203         }
204 
205         /*
206          * Set a new TLS for the child thread?
207          */
208         if (clone_flags & CLONE_SETTLS) {
209 #ifdef CONFIG_IA32_EMULATION
210                 if (test_thread_flag(TIF_IA32))
211                         err = do_set_thread_area(p, -1,
212                                 (struct user_desc __user *)childregs->si, 0);
213                 else
214 #endif
215                         err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
216                 if (err)
217                         goto out;
218         }
219         err = 0;
220 out:
221         if (err && p->thread.io_bitmap_ptr) {
222                 kfree(p->thread.io_bitmap_ptr);
223                 p->thread.io_bitmap_max = 0;
224         }
225 
226         return err;
227 }
228 
229 static void
230 start_thread_common(struct pt_regs *regs, unsigned long new_ip,
231                     unsigned long new_sp,
232                     unsigned int _cs, unsigned int _ss, unsigned int _ds)
233 {
234         loadsegment(fs, 0);
235         loadsegment(es, _ds);
236         loadsegment(ds, _ds);
237         load_gs_index(0);
238         current->thread.usersp  = new_sp;
239         regs->ip                = new_ip;
240         regs->sp                = new_sp;
241         this_cpu_write(old_rsp, new_sp);
242         regs->cs                = _cs;
243         regs->ss                = _ss;
244         regs->flags             = X86_EFLAGS_IF;
245 }
246 
247 void
248 start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
249 {
250         start_thread_common(regs, new_ip, new_sp,
251                             __USER_CS, __USER_DS, 0);
252 }
253 
254 #ifdef CONFIG_IA32_EMULATION
255 void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp)
256 {
257         start_thread_common(regs, new_ip, new_sp,
258                             test_thread_flag(TIF_X32)
259                             ? __USER_CS : __USER32_CS,
260                             __USER_DS, __USER_DS);
261 }
262 #endif
263 
264 /*
265  *      switch_to(x,y) should switch tasks from x to y.
266  *
267  * This could still be optimized:
268  * - fold all the options into a flag word and test it with a single test.
269  * - could test fs/gs bitsliced
270  *
271  * Kprobes not supported here. Set the probe on schedule instead.
272  * Function graph tracer not supported too.
273  */
274 __visible __notrace_funcgraph struct task_struct *
275 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
276 {
277         struct thread_struct *prev = &prev_p->thread;
278         struct thread_struct *next = &next_p->thread;
279         int cpu = smp_processor_id();
280         struct tss_struct *tss = &per_cpu(init_tss, cpu);
281         unsigned fsindex, gsindex;
282         fpu_switch_t fpu;
283 
284         fpu = switch_fpu_prepare(prev_p, next_p, cpu);
285 
286         /* Reload esp0 and ss1. */
287         load_sp0(tss, next);
288 
289         /* We must save %fs and %gs before load_TLS() because
290          * %fs and %gs may be cleared by load_TLS().
291          *
292          * (e.g. xen_load_tls())
293          */
294         savesegment(fs, fsindex);
295         savesegment(gs, gsindex);
296 
297         /*
298          * Load TLS before restoring any segments so that segment loads
299          * reference the correct GDT entries.
300          */
301         load_TLS(next, cpu);
302 
303         /*
304          * Leave lazy mode, flushing any hypercalls made here.  This
305          * must be done after loading TLS entries in the GDT but before
306          * loading segments that might reference them, and and it must
307          * be done before math_state_restore, so the TS bit is up to
308          * date.
309          */
310         arch_end_context_switch(next_p);
311 
312         /* Switch DS and ES.
313          *
314          * Reading them only returns the selectors, but writing them (if
315          * nonzero) loads the full descriptor from the GDT or LDT.  The
316          * LDT for next is loaded in switch_mm, and the GDT is loaded
317          * above.
318          *
319          * We therefore need to write new values to the segment
320          * registers on every context switch unless both the new and old
321          * values are zero.
322          *
323          * Note that we don't need to do anything for CS and SS, as
324          * those are saved and restored as part of pt_regs.
325          */
326         savesegment(es, prev->es);
327         if (unlikely(next->es | prev->es))
328                 loadsegment(es, next->es);
329 
330         savesegment(ds, prev->ds);
331         if (unlikely(next->ds | prev->ds))
332                 loadsegment(ds, next->ds);
333 
334         /*
335          * Switch FS and GS.
336          *
337          * These are even more complicated than FS and GS: they have
338          * 64-bit bases are that controlled by arch_prctl.  Those bases
339          * only differ from the values in the GDT or LDT if the selector
340          * is 0.
341          *
342          * Loading the segment register resets the hidden base part of
343          * the register to 0 or the value from the GDT / LDT.  If the
344          * next base address zero, writing 0 to the segment register is
345          * much faster than using wrmsr to explicitly zero the base.
346          *
347          * The thread_struct.fs and thread_struct.gs values are 0
348          * if the fs and gs bases respectively are not overridden
349          * from the values implied by fsindex and gsindex.  They
350          * are nonzero, and store the nonzero base addresses, if
351          * the bases are overridden.
352          *
353          * (fs != 0 && fsindex != 0) || (gs != 0 && gsindex != 0) should
354          * be impossible.
355          *
356          * Therefore we need to reload the segment registers if either
357          * the old or new selector is nonzero, and we need to override
358          * the base address if next thread expects it to be overridden.
359          *
360          * This code is unnecessarily slow in the case where the old and
361          * new indexes are zero and the new base is nonzero -- it will
362          * unnecessarily write 0 to the selector before writing the new
363          * base address.
364          *
365          * Note: This all depends on arch_prctl being the only way that
366          * user code can override the segment base.  Once wrfsbase and
367          * wrgsbase are enabled, most of this code will need to change.
368          */
369         if (unlikely(fsindex | next->fsindex | prev->fs)) {
370                 loadsegment(fs, next->fsindex);
371 
372                 /*
373                  * If user code wrote a nonzero value to FS, then it also
374                  * cleared the overridden base address.
375                  *
376                  * XXX: if user code wrote 0 to FS and cleared the base
377                  * address itself, we won't notice and we'll incorrectly
378                  * restore the prior base address next time we reschdule
379                  * the process.
380                  */
381                 if (fsindex)
382                         prev->fs = 0;
383         }
384         if (next->fs)
385                 wrmsrl(MSR_FS_BASE, next->fs);
386         prev->fsindex = fsindex;
387 
388         if (unlikely(gsindex | next->gsindex | prev->gs)) {
389                 load_gs_index(next->gsindex);
390 
391                 /* This works (and fails) the same way as fsindex above. */
392                 if (gsindex)
393                         prev->gs = 0;
394         }
395         if (next->gs)
396                 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
397         prev->gsindex = gsindex;
398 
399         switch_fpu_finish(next_p, fpu);
400 
401         /*
402          * Switch the PDA and FPU contexts.
403          */
404         prev->usersp = this_cpu_read(old_rsp);
405         this_cpu_write(old_rsp, next->usersp);
406         this_cpu_write(current_task, next_p);
407 
408         /*
409          * If it were not for PREEMPT_ACTIVE we could guarantee that the
410          * preempt_count of all tasks was equal here and this would not be
411          * needed.
412          */
413         task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
414         this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);
415 
416         this_cpu_write(kernel_stack,
417                   (unsigned long)task_stack_page(next_p) +
418                   THREAD_SIZE - KERNEL_STACK_OFFSET);
419 
420         /*
421          * Now maybe reload the debug registers and handle I/O bitmaps
422          */
423         if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
424                      task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
425                 __switch_to_xtra(prev_p, next_p, tss);
426 
427         return prev_p;
428 }
429 
430 void set_personality_64bit(void)
431 {
432         /* inherit personality from parent */
433 
434         /* Make sure to be in 64bit mode */
435         clear_thread_flag(TIF_IA32);
436         clear_thread_flag(TIF_ADDR32);
437         clear_thread_flag(TIF_X32);
438 
439         /* Ensure the corresponding mm is not marked. */
440         if (current->mm)
441                 current->mm->context.ia32_compat = 0;
442 
443         /* TBD: overwrites user setup. Should have two bits.
444            But 64bit processes have always behaved this way,
445            so it's not too bad. The main problem is just that
446            32bit childs are affected again. */
447         current->personality &= ~READ_IMPLIES_EXEC;
448 }
449 
450 void set_personality_ia32(bool x32)
451 {
452         /* inherit personality from parent */
453 
454         /* Make sure to be in 32bit mode */
455         set_thread_flag(TIF_ADDR32);
456 
457         /* Mark the associated mm as containing 32-bit tasks. */
458         if (x32) {
459                 clear_thread_flag(TIF_IA32);
460                 set_thread_flag(TIF_X32);
461                 if (current->mm)
462                         current->mm->context.ia32_compat = TIF_X32;
463                 current->personality &= ~READ_IMPLIES_EXEC;
464                 /* is_compat_task() uses the presence of the x32
465                    syscall bit flag to determine compat status */
466                 current_thread_info()->status &= ~TS_COMPAT;
467         } else {
468                 set_thread_flag(TIF_IA32);
469                 clear_thread_flag(TIF_X32);
470                 if (current->mm)
471                         current->mm->context.ia32_compat = TIF_IA32;
472                 current->personality |= force_personality32;
473                 /* Prepare the first "return" to user space */
474                 current_thread_info()->status |= TS_COMPAT;
475         }
476 }
477 EXPORT_SYMBOL_GPL(set_personality_ia32);
478 
479 unsigned long get_wchan(struct task_struct *p)
480 {
481         unsigned long stack;
482         u64 fp, ip;
483         int count = 0;
484 
485         if (!p || p == current || p->state == TASK_RUNNING)
486                 return 0;
487         stack = (unsigned long)task_stack_page(p);
488         if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
489                 return 0;
490         fp = *(u64 *)(p->thread.sp);
491         do {
492                 if (fp < (unsigned long)stack ||
493                     fp >= (unsigned long)stack+THREAD_SIZE)
494                         return 0;
495                 ip = *(u64 *)(fp+8);
496                 if (!in_sched_functions(ip))
497                         return ip;
498                 fp = *(u64 *)fp;
499         } while (count++ < 16);
500         return 0;
501 }
502 
503 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
504 {
505         int ret = 0;
506         int doit = task == current;
507         int cpu;
508 
509         switch (code) {
510         case ARCH_SET_GS:
511                 if (addr >= TASK_SIZE_OF(task))
512                         return -EPERM;
513                 cpu = get_cpu();
514                 /* handle small bases via the GDT because that's faster to
515                    switch. */
516                 if (addr <= 0xffffffff) {
517                         set_32bit_tls(task, GS_TLS, addr);
518                         if (doit) {
519                                 load_TLS(&task->thread, cpu);
520                                 load_gs_index(GS_TLS_SEL);
521                         }
522                         task->thread.gsindex = GS_TLS_SEL;
523                         task->thread.gs = 0;
524                 } else {
525                         task->thread.gsindex = 0;
526                         task->thread.gs = addr;
527                         if (doit) {
528                                 load_gs_index(0);
529                                 ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr);
530                         }
531                 }
532                 put_cpu();
533                 break;
534         case ARCH_SET_FS:
535                 /* Not strictly needed for fs, but do it for symmetry
536                    with gs */
537                 if (addr >= TASK_SIZE_OF(task))
538                         return -EPERM;
539                 cpu = get_cpu();
540                 /* handle small bases via the GDT because that's faster to
541                    switch. */
542                 if (addr <= 0xffffffff) {
543                         set_32bit_tls(task, FS_TLS, addr);
544                         if (doit) {
545                                 load_TLS(&task->thread, cpu);
546                                 loadsegment(fs, FS_TLS_SEL);
547                         }
548                         task->thread.fsindex = FS_TLS_SEL;
549                         task->thread.fs = 0;
550                 } else {
551                         task->thread.fsindex = 0;
552                         task->thread.fs = addr;
553                         if (doit) {
554                                 /* set the selector to 0 to not confuse
555                                    __switch_to */
556                                 loadsegment(fs, 0);
557                                 ret = wrmsrl_safe(MSR_FS_BASE, addr);
558                         }
559                 }
560                 put_cpu();
561                 break;
562         case ARCH_GET_FS: {
563                 unsigned long base;
564                 if (task->thread.fsindex == FS_TLS_SEL)
565                         base = read_32bit_tls(task, FS_TLS);
566                 else if (doit)
567                         rdmsrl(MSR_FS_BASE, base);
568                 else
569                         base = task->thread.fs;
570                 ret = put_user(base, (unsigned long __user *)addr);
571                 break;
572         }
573         case ARCH_GET_GS: {
574                 unsigned long base;
575                 unsigned gsindex;
576                 if (task->thread.gsindex == GS_TLS_SEL)
577                         base = read_32bit_tls(task, GS_TLS);
578                 else if (doit) {
579                         savesegment(gs, gsindex);
580                         if (gsindex)
581                                 rdmsrl(MSR_KERNEL_GS_BASE, base);
582                         else
583                                 base = task->thread.gs;
584                 } else
585                         base = task->thread.gs;
586                 ret = put_user(base, (unsigned long __user *)addr);
587                 break;
588         }
589 
590         default:
591                 ret = -EINVAL;
592                 break;
593         }
594 
595         return ret;
596 }
597 
598 long sys_arch_prctl(int code, unsigned long addr)
599 {
600         return do_arch_prctl(current, code, addr);
601 }
602 
603 unsigned long KSTK_ESP(struct task_struct *task)
604 {
605         return (test_tsk_thread_flag(task, TIF_IA32)) ?
606                         (task_pt_regs(task)->sp) : ((task)->thread.usersp);
607 }
608 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us