Version:  2.0.40 2.2.26 2.4.37 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6

Linux/kernel/irq/manage.c

  1 /*
  2  * linux/kernel/irq/manage.c
  3  *
  4  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
  5  * Copyright (C) 2005-2006 Thomas Gleixner
  6  *
  7  * This file contains driver APIs to the irq subsystem.
  8  */
  9 
 10 #define pr_fmt(fmt) "genirq: " fmt
 11 
 12 #include <linux/irq.h>
 13 #include <linux/kthread.h>
 14 #include <linux/module.h>
 15 #include <linux/random.h>
 16 #include <linux/interrupt.h>
 17 #include <linux/slab.h>
 18 #include <linux/sched.h>
 19 #include <linux/sched/rt.h>
 20 #include <linux/task_work.h>
 21 
 22 #include "internals.h"
 23 
 24 #ifdef CONFIG_IRQ_FORCED_THREADING
 25 __read_mostly bool force_irqthreads;
 26 
 27 static int __init setup_forced_irqthreads(char *arg)
 28 {
 29         force_irqthreads = true;
 30         return 0;
 31 }
 32 early_param("threadirqs", setup_forced_irqthreads);
 33 #endif
 34 
 35 static void __synchronize_hardirq(struct irq_desc *desc)
 36 {
 37         bool inprogress;
 38 
 39         do {
 40                 unsigned long flags;
 41 
 42                 /*
 43                  * Wait until we're out of the critical section.  This might
 44                  * give the wrong answer due to the lack of memory barriers.
 45                  */
 46                 while (irqd_irq_inprogress(&desc->irq_data))
 47                         cpu_relax();
 48 
 49                 /* Ok, that indicated we're done: double-check carefully. */
 50                 raw_spin_lock_irqsave(&desc->lock, flags);
 51                 inprogress = irqd_irq_inprogress(&desc->irq_data);
 52                 raw_spin_unlock_irqrestore(&desc->lock, flags);
 53 
 54                 /* Oops, that failed? */
 55         } while (inprogress);
 56 }
 57 
 58 /**
 59  *      synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
 60  *      @irq: interrupt number to wait for
 61  *
 62  *      This function waits for any pending hard IRQ handlers for this
 63  *      interrupt to complete before returning. If you use this
 64  *      function while holding a resource the IRQ handler may need you
 65  *      will deadlock. It does not take associated threaded handlers
 66  *      into account.
 67  *
 68  *      Do not use this for shutdown scenarios where you must be sure
 69  *      that all parts (hardirq and threaded handler) have completed.
 70  *
 71  *      Returns: false if a threaded handler is active.
 72  *
 73  *      This function may be called - with care - from IRQ context.
 74  */
 75 bool synchronize_hardirq(unsigned int irq)
 76 {
 77         struct irq_desc *desc = irq_to_desc(irq);
 78 
 79         if (desc) {
 80                 __synchronize_hardirq(desc);
 81                 return !atomic_read(&desc->threads_active);
 82         }
 83 
 84         return true;
 85 }
 86 EXPORT_SYMBOL(synchronize_hardirq);
 87 
 88 /**
 89  *      synchronize_irq - wait for pending IRQ handlers (on other CPUs)
 90  *      @irq: interrupt number to wait for
 91  *
 92  *      This function waits for any pending IRQ handlers for this interrupt
 93  *      to complete before returning. If you use this function while
 94  *      holding a resource the IRQ handler may need you will deadlock.
 95  *
 96  *      This function may be called - with care - from IRQ context.
 97  */
 98 void synchronize_irq(unsigned int irq)
 99 {
100         struct irq_desc *desc = irq_to_desc(irq);
101 
102         if (desc) {
103                 __synchronize_hardirq(desc);
104                 /*
105                  * We made sure that no hardirq handler is
106                  * running. Now verify that no threaded handlers are
107                  * active.
108                  */
109                 wait_event(desc->wait_for_threads,
110                            !atomic_read(&desc->threads_active));
111         }
112 }
113 EXPORT_SYMBOL(synchronize_irq);
114 
115 #ifdef CONFIG_SMP
116 cpumask_var_t irq_default_affinity;
117 
118 static int __irq_can_set_affinity(struct irq_desc *desc)
119 {
120         if (!desc || !irqd_can_balance(&desc->irq_data) ||
121             !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
122                 return 0;
123         return 1;
124 }
125 
126 /**
127  *      irq_can_set_affinity - Check if the affinity of a given irq can be set
128  *      @irq:           Interrupt to check
129  *
130  */
131 int irq_can_set_affinity(unsigned int irq)
132 {
133         return __irq_can_set_affinity(irq_to_desc(irq));
134 }
135 
136 /**
137  *      irq_set_thread_affinity - Notify irq threads to adjust affinity
138  *      @desc:          irq descriptor which has affitnity changed
139  *
140  *      We just set IRQTF_AFFINITY and delegate the affinity setting
141  *      to the interrupt thread itself. We can not call
142  *      set_cpus_allowed_ptr() here as we hold desc->lock and this
143  *      code can be called from hard interrupt context.
144  */
145 void irq_set_thread_affinity(struct irq_desc *desc)
146 {
147         struct irqaction *action;
148 
149         for_each_action_of_desc(desc, action)
150                 if (action->thread)
151                         set_bit(IRQTF_AFFINITY, &action->thread_flags);
152 }
153 
154 #ifdef CONFIG_GENERIC_PENDING_IRQ
155 static inline bool irq_can_move_pcntxt(struct irq_data *data)
156 {
157         return irqd_can_move_in_process_context(data);
158 }
159 static inline bool irq_move_pending(struct irq_data *data)
160 {
161         return irqd_is_setaffinity_pending(data);
162 }
163 static inline void
164 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
165 {
166         cpumask_copy(desc->pending_mask, mask);
167 }
168 static inline void
169 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
170 {
171         cpumask_copy(mask, desc->pending_mask);
172 }
173 #else
174 static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
175 static inline bool irq_move_pending(struct irq_data *data) { return false; }
176 static inline void
177 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
178 static inline void
179 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
180 #endif
181 
182 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
183                         bool force)
184 {
185         struct irq_desc *desc = irq_data_to_desc(data);
186         struct irq_chip *chip = irq_data_get_irq_chip(data);
187         int ret;
188 
189         ret = chip->irq_set_affinity(data, mask, force);
190         switch (ret) {
191         case IRQ_SET_MASK_OK:
192         case IRQ_SET_MASK_OK_DONE:
193                 cpumask_copy(desc->irq_common_data.affinity, mask);
194         case IRQ_SET_MASK_OK_NOCOPY:
195                 irq_set_thread_affinity(desc);
196                 ret = 0;
197         }
198 
199         return ret;
200 }
201 
202 int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
203                             bool force)
204 {
205         struct irq_chip *chip = irq_data_get_irq_chip(data);
206         struct irq_desc *desc = irq_data_to_desc(data);
207         int ret = 0;
208 
209         if (!chip || !chip->irq_set_affinity)
210                 return -EINVAL;
211 
212         if (irq_can_move_pcntxt(data)) {
213                 ret = irq_do_set_affinity(data, mask, force);
214         } else {
215                 irqd_set_move_pending(data);
216                 irq_copy_pending(desc, mask);
217         }
218 
219         if (desc->affinity_notify) {
220                 kref_get(&desc->affinity_notify->kref);
221                 schedule_work(&desc->affinity_notify->work);
222         }
223         irqd_set(data, IRQD_AFFINITY_SET);
224 
225         return ret;
226 }
227 
228 int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
229 {
230         struct irq_desc *desc = irq_to_desc(irq);
231         unsigned long flags;
232         int ret;
233 
234         if (!desc)
235                 return -EINVAL;
236 
237         raw_spin_lock_irqsave(&desc->lock, flags);
238         ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
239         raw_spin_unlock_irqrestore(&desc->lock, flags);
240         return ret;
241 }
242 
243 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
244 {
245         unsigned long flags;
246         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
247 
248         if (!desc)
249                 return -EINVAL;
250         desc->affinity_hint = m;
251         irq_put_desc_unlock(desc, flags);
252         /* set the initial affinity to prevent every interrupt being on CPU0 */
253         if (m)
254                 __irq_set_affinity(irq, m, false);
255         return 0;
256 }
257 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
258 
259 static void irq_affinity_notify(struct work_struct *work)
260 {
261         struct irq_affinity_notify *notify =
262                 container_of(work, struct irq_affinity_notify, work);
263         struct irq_desc *desc = irq_to_desc(notify->irq);
264         cpumask_var_t cpumask;
265         unsigned long flags;
266 
267         if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
268                 goto out;
269 
270         raw_spin_lock_irqsave(&desc->lock, flags);
271         if (irq_move_pending(&desc->irq_data))
272                 irq_get_pending(cpumask, desc);
273         else
274                 cpumask_copy(cpumask, desc->irq_common_data.affinity);
275         raw_spin_unlock_irqrestore(&desc->lock, flags);
276 
277         notify->notify(notify, cpumask);
278 
279         free_cpumask_var(cpumask);
280 out:
281         kref_put(&notify->kref, notify->release);
282 }
283 
284 /**
285  *      irq_set_affinity_notifier - control notification of IRQ affinity changes
286  *      @irq:           Interrupt for which to enable/disable notification
287  *      @notify:        Context for notification, or %NULL to disable
288  *                      notification.  Function pointers must be initialised;
289  *                      the other fields will be initialised by this function.
290  *
291  *      Must be called in process context.  Notification may only be enabled
292  *      after the IRQ is allocated and must be disabled before the IRQ is
293  *      freed using free_irq().
294  */
295 int
296 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
297 {
298         struct irq_desc *desc = irq_to_desc(irq);
299         struct irq_affinity_notify *old_notify;
300         unsigned long flags;
301 
302         /* The release function is promised process context */
303         might_sleep();
304 
305         if (!desc)
306                 return -EINVAL;
307 
308         /* Complete initialisation of *notify */
309         if (notify) {
310                 notify->irq = irq;
311                 kref_init(&notify->kref);
312                 INIT_WORK(&notify->work, irq_affinity_notify);
313         }
314 
315         raw_spin_lock_irqsave(&desc->lock, flags);
316         old_notify = desc->affinity_notify;
317         desc->affinity_notify = notify;
318         raw_spin_unlock_irqrestore(&desc->lock, flags);
319 
320         if (old_notify)
321                 kref_put(&old_notify->kref, old_notify->release);
322 
323         return 0;
324 }
325 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
326 
327 #ifndef CONFIG_AUTO_IRQ_AFFINITY
328 /*
329  * Generic version of the affinity autoselector.
330  */
331 static int setup_affinity(struct irq_desc *desc, struct cpumask *mask)
332 {
333         struct cpumask *set = irq_default_affinity;
334         int node = irq_desc_get_node(desc);
335 
336         /* Excludes PER_CPU and NO_BALANCE interrupts */
337         if (!__irq_can_set_affinity(desc))
338                 return 0;
339 
340         /*
341          * Preserve an userspace affinity setup, but make sure that
342          * one of the targets is online.
343          */
344         if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
345                 if (cpumask_intersects(desc->irq_common_data.affinity,
346                                        cpu_online_mask))
347                         set = desc->irq_common_data.affinity;
348                 else
349                         irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
350         }
351 
352         cpumask_and(mask, cpu_online_mask, set);
353         if (node != NUMA_NO_NODE) {
354                 const struct cpumask *nodemask = cpumask_of_node(node);
355 
356                 /* make sure at least one of the cpus in nodemask is online */
357                 if (cpumask_intersects(mask, nodemask))
358                         cpumask_and(mask, mask, nodemask);
359         }
360         irq_do_set_affinity(&desc->irq_data, mask, false);
361         return 0;
362 }
363 #else
364 /* Wrapper for ALPHA specific affinity selector magic */
365 static inline int setup_affinity(struct irq_desc *d, struct cpumask *mask)
366 {
367         return irq_select_affinity(irq_desc_get_irq(d));
368 }
369 #endif
370 
371 /*
372  * Called when affinity is set via /proc/irq
373  */
374 int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
375 {
376         struct irq_desc *desc = irq_to_desc(irq);
377         unsigned long flags;
378         int ret;
379 
380         raw_spin_lock_irqsave(&desc->lock, flags);
381         ret = setup_affinity(desc, mask);
382         raw_spin_unlock_irqrestore(&desc->lock, flags);
383         return ret;
384 }
385 
386 #else
387 static inline int
388 setup_affinity(struct irq_desc *desc, struct cpumask *mask)
389 {
390         return 0;
391 }
392 #endif
393 
394 /**
395  *      irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
396  *      @irq: interrupt number to set affinity
397  *      @vcpu_info: vCPU specific data
398  *
399  *      This function uses the vCPU specific data to set the vCPU
400  *      affinity for an irq. The vCPU specific data is passed from
401  *      outside, such as KVM. One example code path is as below:
402  *      KVM -> IOMMU -> irq_set_vcpu_affinity().
403  */
404 int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
405 {
406         unsigned long flags;
407         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
408         struct irq_data *data;
409         struct irq_chip *chip;
410         int ret = -ENOSYS;
411 
412         if (!desc)
413                 return -EINVAL;
414 
415         data = irq_desc_get_irq_data(desc);
416         chip = irq_data_get_irq_chip(data);
417         if (chip && chip->irq_set_vcpu_affinity)
418                 ret = chip->irq_set_vcpu_affinity(data, vcpu_info);
419         irq_put_desc_unlock(desc, flags);
420 
421         return ret;
422 }
423 EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
424 
425 void __disable_irq(struct irq_desc *desc)
426 {
427         if (!desc->depth++)
428                 irq_disable(desc);
429 }
430 
431 static int __disable_irq_nosync(unsigned int irq)
432 {
433         unsigned long flags;
434         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
435 
436         if (!desc)
437                 return -EINVAL;
438         __disable_irq(desc);
439         irq_put_desc_busunlock(desc, flags);
440         return 0;
441 }
442 
443 /**
444  *      disable_irq_nosync - disable an irq without waiting
445  *      @irq: Interrupt to disable
446  *
447  *      Disable the selected interrupt line.  Disables and Enables are
448  *      nested.
449  *      Unlike disable_irq(), this function does not ensure existing
450  *      instances of the IRQ handler have completed before returning.
451  *
452  *      This function may be called from IRQ context.
453  */
454 void disable_irq_nosync(unsigned int irq)
455 {
456         __disable_irq_nosync(irq);
457 }
458 EXPORT_SYMBOL(disable_irq_nosync);
459 
460 /**
461  *      disable_irq - disable an irq and wait for completion
462  *      @irq: Interrupt to disable
463  *
464  *      Disable the selected interrupt line.  Enables and Disables are
465  *      nested.
466  *      This function waits for any pending IRQ handlers for this interrupt
467  *      to complete before returning. If you use this function while
468  *      holding a resource the IRQ handler may need you will deadlock.
469  *
470  *      This function may be called - with care - from IRQ context.
471  */
472 void disable_irq(unsigned int irq)
473 {
474         if (!__disable_irq_nosync(irq))
475                 synchronize_irq(irq);
476 }
477 EXPORT_SYMBOL(disable_irq);
478 
479 /**
480  *      disable_hardirq - disables an irq and waits for hardirq completion
481  *      @irq: Interrupt to disable
482  *
483  *      Disable the selected interrupt line.  Enables and Disables are
484  *      nested.
485  *      This function waits for any pending hard IRQ handlers for this
486  *      interrupt to complete before returning. If you use this function while
487  *      holding a resource the hard IRQ handler may need you will deadlock.
488  *
489  *      When used to optimistically disable an interrupt from atomic context
490  *      the return value must be checked.
491  *
492  *      Returns: false if a threaded handler is active.
493  *
494  *      This function may be called - with care - from IRQ context.
495  */
496 bool disable_hardirq(unsigned int irq)
497 {
498         if (!__disable_irq_nosync(irq))
499                 return synchronize_hardirq(irq);
500 
501         return false;
502 }
503 EXPORT_SYMBOL_GPL(disable_hardirq);
504 
505 void __enable_irq(struct irq_desc *desc)
506 {
507         switch (desc->depth) {
508         case 0:
509  err_out:
510                 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
511                      irq_desc_get_irq(desc));
512                 break;
513         case 1: {
514                 if (desc->istate & IRQS_SUSPENDED)
515                         goto err_out;
516                 /* Prevent probing on this irq: */
517                 irq_settings_set_noprobe(desc);
518                 irq_enable(desc);
519                 check_irq_resend(desc);
520                 /* fall-through */
521         }
522         default:
523                 desc->depth--;
524         }
525 }
526 
527 /**
528  *      enable_irq - enable handling of an irq
529  *      @irq: Interrupt to enable
530  *
531  *      Undoes the effect of one call to disable_irq().  If this
532  *      matches the last disable, processing of interrupts on this
533  *      IRQ line is re-enabled.
534  *
535  *      This function may be called from IRQ context only when
536  *      desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
537  */
538 void enable_irq(unsigned int irq)
539 {
540         unsigned long flags;
541         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
542 
543         if (!desc)
544                 return;
545         if (WARN(!desc->irq_data.chip,
546                  KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
547                 goto out;
548 
549         __enable_irq(desc);
550 out:
551         irq_put_desc_busunlock(desc, flags);
552 }
553 EXPORT_SYMBOL(enable_irq);
554 
555 static int set_irq_wake_real(unsigned int irq, unsigned int on)
556 {
557         struct irq_desc *desc = irq_to_desc(irq);
558         int ret = -ENXIO;
559 
560         if (irq_desc_get_chip(desc)->flags &  IRQCHIP_SKIP_SET_WAKE)
561                 return 0;
562 
563         if (desc->irq_data.chip->irq_set_wake)
564                 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
565 
566         return ret;
567 }
568 
569 /**
570  *      irq_set_irq_wake - control irq power management wakeup
571  *      @irq:   interrupt to control
572  *      @on:    enable/disable power management wakeup
573  *
574  *      Enable/disable power management wakeup mode, which is
575  *      disabled by default.  Enables and disables must match,
576  *      just as they match for non-wakeup mode support.
577  *
578  *      Wakeup mode lets this IRQ wake the system from sleep
579  *      states like "suspend to RAM".
580  */
581 int irq_set_irq_wake(unsigned int irq, unsigned int on)
582 {
583         unsigned long flags;
584         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
585         int ret = 0;
586 
587         if (!desc)
588                 return -EINVAL;
589 
590         /* wakeup-capable irqs can be shared between drivers that
591          * don't need to have the same sleep mode behaviors.
592          */
593         if (on) {
594                 if (desc->wake_depth++ == 0) {
595                         ret = set_irq_wake_real(irq, on);
596                         if (ret)
597                                 desc->wake_depth = 0;
598                         else
599                                 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
600                 }
601         } else {
602                 if (desc->wake_depth == 0) {
603                         WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
604                 } else if (--desc->wake_depth == 0) {
605                         ret = set_irq_wake_real(irq, on);
606                         if (ret)
607                                 desc->wake_depth = 1;
608                         else
609                                 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
610                 }
611         }
612         irq_put_desc_busunlock(desc, flags);
613         return ret;
614 }
615 EXPORT_SYMBOL(irq_set_irq_wake);
616 
617 /*
618  * Internal function that tells the architecture code whether a
619  * particular irq has been exclusively allocated or is available
620  * for driver use.
621  */
622 int can_request_irq(unsigned int irq, unsigned long irqflags)
623 {
624         unsigned long flags;
625         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
626         int canrequest = 0;
627 
628         if (!desc)
629                 return 0;
630 
631         if (irq_settings_can_request(desc)) {
632                 if (!desc->action ||
633                     irqflags & desc->action->flags & IRQF_SHARED)
634                         canrequest = 1;
635         }
636         irq_put_desc_unlock(desc, flags);
637         return canrequest;
638 }
639 
640 int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
641 {
642         struct irq_chip *chip = desc->irq_data.chip;
643         int ret, unmask = 0;
644 
645         if (!chip || !chip->irq_set_type) {
646                 /*
647                  * IRQF_TRIGGER_* but the PIC does not support multiple
648                  * flow-types?
649                  */
650                 pr_debug("No set_type function for IRQ %d (%s)\n",
651                          irq_desc_get_irq(desc),
652                          chip ? (chip->name ? : "unknown") : "unknown");
653                 return 0;
654         }
655 
656         flags &= IRQ_TYPE_SENSE_MASK;
657 
658         if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
659                 if (!irqd_irq_masked(&desc->irq_data))
660                         mask_irq(desc);
661                 if (!irqd_irq_disabled(&desc->irq_data))
662                         unmask = 1;
663         }
664 
665         /* caller masked out all except trigger mode flags */
666         ret = chip->irq_set_type(&desc->irq_data, flags);
667 
668         switch (ret) {
669         case IRQ_SET_MASK_OK:
670         case IRQ_SET_MASK_OK_DONE:
671                 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
672                 irqd_set(&desc->irq_data, flags);
673 
674         case IRQ_SET_MASK_OK_NOCOPY:
675                 flags = irqd_get_trigger_type(&desc->irq_data);
676                 irq_settings_set_trigger_mask(desc, flags);
677                 irqd_clear(&desc->irq_data, IRQD_LEVEL);
678                 irq_settings_clr_level(desc);
679                 if (flags & IRQ_TYPE_LEVEL_MASK) {
680                         irq_settings_set_level(desc);
681                         irqd_set(&desc->irq_data, IRQD_LEVEL);
682                 }
683 
684                 ret = 0;
685                 break;
686         default:
687                 pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
688                        flags, irq_desc_get_irq(desc), chip->irq_set_type);
689         }
690         if (unmask)
691                 unmask_irq(desc);
692         return ret;
693 }
694 
695 #ifdef CONFIG_HARDIRQS_SW_RESEND
696 int irq_set_parent(int irq, int parent_irq)
697 {
698         unsigned long flags;
699         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
700 
701         if (!desc)
702                 return -EINVAL;
703 
704         desc->parent_irq = parent_irq;
705 
706         irq_put_desc_unlock(desc, flags);
707         return 0;
708 }
709 #endif
710 
711 /*
712  * Default primary interrupt handler for threaded interrupts. Is
713  * assigned as primary handler when request_threaded_irq is called
714  * with handler == NULL. Useful for oneshot interrupts.
715  */
716 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
717 {
718         return IRQ_WAKE_THREAD;
719 }
720 
721 /*
722  * Primary handler for nested threaded interrupts. Should never be
723  * called.
724  */
725 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
726 {
727         WARN(1, "Primary handler called for nested irq %d\n", irq);
728         return IRQ_NONE;
729 }
730 
731 static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
732 {
733         WARN(1, "Secondary action handler called for irq %d\n", irq);
734         return IRQ_NONE;
735 }
736 
737 static int irq_wait_for_interrupt(struct irqaction *action)
738 {
739         set_current_state(TASK_INTERRUPTIBLE);
740 
741         while (!kthread_should_stop()) {
742 
743                 if (test_and_clear_bit(IRQTF_RUNTHREAD,
744                                        &action->thread_flags)) {
745                         __set_current_state(TASK_RUNNING);
746                         return 0;
747                 }
748                 schedule();
749                 set_current_state(TASK_INTERRUPTIBLE);
750         }
751         __set_current_state(TASK_RUNNING);
752         return -1;
753 }
754 
755 /*
756  * Oneshot interrupts keep the irq line masked until the threaded
757  * handler finished. unmask if the interrupt has not been disabled and
758  * is marked MASKED.
759  */
760 static void irq_finalize_oneshot(struct irq_desc *desc,
761                                  struct irqaction *action)
762 {
763         if (!(desc->istate & IRQS_ONESHOT) ||
764             action->handler == irq_forced_secondary_handler)
765                 return;
766 again:
767         chip_bus_lock(desc);
768         raw_spin_lock_irq(&desc->lock);
769 
770         /*
771          * Implausible though it may be we need to protect us against
772          * the following scenario:
773          *
774          * The thread is faster done than the hard interrupt handler
775          * on the other CPU. If we unmask the irq line then the
776          * interrupt can come in again and masks the line, leaves due
777          * to IRQS_INPROGRESS and the irq line is masked forever.
778          *
779          * This also serializes the state of shared oneshot handlers
780          * versus "desc->threads_onehsot |= action->thread_mask;" in
781          * irq_wake_thread(). See the comment there which explains the
782          * serialization.
783          */
784         if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
785                 raw_spin_unlock_irq(&desc->lock);
786                 chip_bus_sync_unlock(desc);
787                 cpu_relax();
788                 goto again;
789         }
790 
791         /*
792          * Now check again, whether the thread should run. Otherwise
793          * we would clear the threads_oneshot bit of this thread which
794          * was just set.
795          */
796         if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
797                 goto out_unlock;
798 
799         desc->threads_oneshot &= ~action->thread_mask;
800 
801         if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
802             irqd_irq_masked(&desc->irq_data))
803                 unmask_threaded_irq(desc);
804 
805 out_unlock:
806         raw_spin_unlock_irq(&desc->lock);
807         chip_bus_sync_unlock(desc);
808 }
809 
810 #ifdef CONFIG_SMP
811 /*
812  * Check whether we need to change the affinity of the interrupt thread.
813  */
814 static void
815 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
816 {
817         cpumask_var_t mask;
818         bool valid = true;
819 
820         if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
821                 return;
822 
823         /*
824          * In case we are out of memory we set IRQTF_AFFINITY again and
825          * try again next time
826          */
827         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
828                 set_bit(IRQTF_AFFINITY, &action->thread_flags);
829                 return;
830         }
831 
832         raw_spin_lock_irq(&desc->lock);
833         /*
834          * This code is triggered unconditionally. Check the affinity
835          * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
836          */
837         if (desc->irq_common_data.affinity)
838                 cpumask_copy(mask, desc->irq_common_data.affinity);
839         else
840                 valid = false;
841         raw_spin_unlock_irq(&desc->lock);
842 
843         if (valid)
844                 set_cpus_allowed_ptr(current, mask);
845         free_cpumask_var(mask);
846 }
847 #else
848 static inline void
849 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
850 #endif
851 
852 /*
853  * Interrupts which are not explicitely requested as threaded
854  * interrupts rely on the implicit bh/preempt disable of the hard irq
855  * context. So we need to disable bh here to avoid deadlocks and other
856  * side effects.
857  */
858 static irqreturn_t
859 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
860 {
861         irqreturn_t ret;
862 
863         local_bh_disable();
864         ret = action->thread_fn(action->irq, action->dev_id);
865         irq_finalize_oneshot(desc, action);
866         local_bh_enable();
867         return ret;
868 }
869 
870 /*
871  * Interrupts explicitly requested as threaded interrupts want to be
872  * preemtible - many of them need to sleep and wait for slow busses to
873  * complete.
874  */
875 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
876                 struct irqaction *action)
877 {
878         irqreturn_t ret;
879 
880         ret = action->thread_fn(action->irq, action->dev_id);
881         irq_finalize_oneshot(desc, action);
882         return ret;
883 }
884 
885 static void wake_threads_waitq(struct irq_desc *desc)
886 {
887         if (atomic_dec_and_test(&desc->threads_active))
888                 wake_up(&desc->wait_for_threads);
889 }
890 
891 static void irq_thread_dtor(struct callback_head *unused)
892 {
893         struct task_struct *tsk = current;
894         struct irq_desc *desc;
895         struct irqaction *action;
896 
897         if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
898                 return;
899 
900         action = kthread_data(tsk);
901 
902         pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
903                tsk->comm, tsk->pid, action->irq);
904 
905 
906         desc = irq_to_desc(action->irq);
907         /*
908          * If IRQTF_RUNTHREAD is set, we need to decrement
909          * desc->threads_active and wake possible waiters.
910          */
911         if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
912                 wake_threads_waitq(desc);
913 
914         /* Prevent a stale desc->threads_oneshot */
915         irq_finalize_oneshot(desc, action);
916 }
917 
918 static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
919 {
920         struct irqaction *secondary = action->secondary;
921 
922         if (WARN_ON_ONCE(!secondary))
923                 return;
924 
925         raw_spin_lock_irq(&desc->lock);
926         __irq_wake_thread(desc, secondary);
927         raw_spin_unlock_irq(&desc->lock);
928 }
929 
930 /*
931  * Interrupt handler thread
932  */
933 static int irq_thread(void *data)
934 {
935         struct callback_head on_exit_work;
936         struct irqaction *action = data;
937         struct irq_desc *desc = irq_to_desc(action->irq);
938         irqreturn_t (*handler_fn)(struct irq_desc *desc,
939                         struct irqaction *action);
940 
941         if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
942                                         &action->thread_flags))
943                 handler_fn = irq_forced_thread_fn;
944         else
945                 handler_fn = irq_thread_fn;
946 
947         init_task_work(&on_exit_work, irq_thread_dtor);
948         task_work_add(current, &on_exit_work, false);
949 
950         irq_thread_check_affinity(desc, action);
951 
952         while (!irq_wait_for_interrupt(action)) {
953                 irqreturn_t action_ret;
954 
955                 irq_thread_check_affinity(desc, action);
956 
957                 action_ret = handler_fn(desc, action);
958                 if (action_ret == IRQ_HANDLED)
959                         atomic_inc(&desc->threads_handled);
960                 if (action_ret == IRQ_WAKE_THREAD)
961                         irq_wake_secondary(desc, action);
962 
963                 wake_threads_waitq(desc);
964         }
965 
966         /*
967          * This is the regular exit path. __free_irq() is stopping the
968          * thread via kthread_stop() after calling
969          * synchronize_irq(). So neither IRQTF_RUNTHREAD nor the
970          * oneshot mask bit can be set. We cannot verify that as we
971          * cannot touch the oneshot mask at this point anymore as
972          * __setup_irq() might have given out currents thread_mask
973          * again.
974          */
975         task_work_cancel(current, irq_thread_dtor);
976         return 0;
977 }
978 
979 /**
980  *      irq_wake_thread - wake the irq thread for the action identified by dev_id
981  *      @irq:           Interrupt line
982  *      @dev_id:        Device identity for which the thread should be woken
983  *
984  */
985 void irq_wake_thread(unsigned int irq, void *dev_id)
986 {
987         struct irq_desc *desc = irq_to_desc(irq);
988         struct irqaction *action;
989         unsigned long flags;
990 
991         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
992                 return;
993 
994         raw_spin_lock_irqsave(&desc->lock, flags);
995         for_each_action_of_desc(desc, action) {
996                 if (action->dev_id == dev_id) {
997                         if (action->thread)
998                                 __irq_wake_thread(desc, action);
999                         break;
1000                 }
1001         }
1002         raw_spin_unlock_irqrestore(&desc->lock, flags);
1003 }
1004 EXPORT_SYMBOL_GPL(irq_wake_thread);
1005 
1006 static int irq_setup_forced_threading(struct irqaction *new)
1007 {
1008         if (!force_irqthreads)
1009                 return 0;
1010         if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1011                 return 0;
1012 
1013         new->flags |= IRQF_ONESHOT;
1014 
1015         /*
1016          * Handle the case where we have a real primary handler and a
1017          * thread handler. We force thread them as well by creating a
1018          * secondary action.
1019          */
1020         if (new->handler != irq_default_primary_handler && new->thread_fn) {
1021                 /* Allocate the secondary action */
1022                 new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1023                 if (!new->secondary)
1024                         return -ENOMEM;
1025                 new->secondary->handler = irq_forced_secondary_handler;
1026                 new->secondary->thread_fn = new->thread_fn;
1027                 new->secondary->dev_id = new->dev_id;
1028                 new->secondary->irq = new->irq;
1029                 new->secondary->name = new->name;
1030         }
1031         /* Deal with the primary handler */
1032         set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
1033         new->thread_fn = new->handler;
1034         new->handler = irq_default_primary_handler;
1035         return 0;
1036 }
1037 
1038 static int irq_request_resources(struct irq_desc *desc)
1039 {
1040         struct irq_data *d = &desc->irq_data;
1041         struct irq_chip *c = d->chip;
1042 
1043         return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1044 }
1045 
1046 static void irq_release_resources(struct irq_desc *desc)
1047 {
1048         struct irq_data *d = &desc->irq_data;
1049         struct irq_chip *c = d->chip;
1050 
1051         if (c->irq_release_resources)
1052                 c->irq_release_resources(d);
1053 }
1054 
1055 static int
1056 setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
1057 {
1058         struct task_struct *t;
1059         struct sched_param param = {
1060                 .sched_priority = MAX_USER_RT_PRIO/2,
1061         };
1062 
1063         if (!secondary) {
1064                 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1065                                    new->name);
1066         } else {
1067                 t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
1068                                    new->name);
1069                 param.sched_priority -= 1;
1070         }
1071 
1072         if (IS_ERR(t))
1073                 return PTR_ERR(t);
1074 
1075         sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
1076 
1077         /*
1078          * We keep the reference to the task struct even if
1079          * the thread dies to avoid that the interrupt code
1080          * references an already freed task_struct.
1081          */
1082         get_task_struct(t);
1083         new->thread = t;
1084         /*
1085          * Tell the thread to set its affinity. This is
1086          * important for shared interrupt handlers as we do
1087          * not invoke setup_affinity() for the secondary
1088          * handlers as everything is already set up. Even for
1089          * interrupts marked with IRQF_NO_BALANCE this is
1090          * correct as we want the thread to move to the cpu(s)
1091          * on which the requesting code placed the interrupt.
1092          */
1093         set_bit(IRQTF_AFFINITY, &new->thread_flags);
1094         return 0;
1095 }
1096 
1097 /*
1098  * Internal function to register an irqaction - typically used to
1099  * allocate special interrupts that are part of the architecture.
1100  */
1101 static int
1102 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1103 {
1104         struct irqaction *old, **old_ptr;
1105         unsigned long flags, thread_mask = 0;
1106         int ret, nested, shared = 0;
1107         cpumask_var_t mask;
1108 
1109         if (!desc)
1110                 return -EINVAL;
1111 
1112         if (desc->irq_data.chip == &no_irq_chip)
1113                 return -ENOSYS;
1114         if (!try_module_get(desc->owner))
1115                 return -ENODEV;
1116 
1117         new->irq = irq;
1118 
1119         /*
1120          * Check whether the interrupt nests into another interrupt
1121          * thread.
1122          */
1123         nested = irq_settings_is_nested_thread(desc);
1124         if (nested) {
1125                 if (!new->thread_fn) {
1126                         ret = -EINVAL;
1127                         goto out_mput;
1128                 }
1129                 /*
1130                  * Replace the primary handler which was provided from
1131                  * the driver for non nested interrupt handling by the
1132                  * dummy function which warns when called.
1133                  */
1134                 new->handler = irq_nested_primary_handler;
1135         } else {
1136                 if (irq_settings_can_thread(desc)) {
1137                         ret = irq_setup_forced_threading(new);
1138                         if (ret)
1139                                 goto out_mput;
1140                 }
1141         }
1142 
1143         /*
1144          * Create a handler thread when a thread function is supplied
1145          * and the interrupt does not nest into another interrupt
1146          * thread.
1147          */
1148         if (new->thread_fn && !nested) {
1149                 ret = setup_irq_thread(new, irq, false);
1150                 if (ret)
1151                         goto out_mput;
1152                 if (new->secondary) {
1153                         ret = setup_irq_thread(new->secondary, irq, true);
1154                         if (ret)
1155                                 goto out_thread;
1156                 }
1157         }
1158 
1159         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1160                 ret = -ENOMEM;
1161                 goto out_thread;
1162         }
1163 
1164         /*
1165          * Drivers are often written to work w/o knowledge about the
1166          * underlying irq chip implementation, so a request for a
1167          * threaded irq without a primary hard irq context handler
1168          * requires the ONESHOT flag to be set. Some irq chips like
1169          * MSI based interrupts are per se one shot safe. Check the
1170          * chip flags, so we can avoid the unmask dance at the end of
1171          * the threaded handler for those.
1172          */
1173         if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1174                 new->flags &= ~IRQF_ONESHOT;
1175 
1176         /*
1177          * The following block of code has to be executed atomically
1178          */
1179         raw_spin_lock_irqsave(&desc->lock, flags);
1180         old_ptr = &desc->action;
1181         old = *old_ptr;
1182         if (old) {
1183                 /*
1184                  * Can't share interrupts unless both agree to and are
1185                  * the same type (level, edge, polarity). So both flag
1186                  * fields must have IRQF_SHARED set and the bits which
1187                  * set the trigger type must match. Also all must
1188                  * agree on ONESHOT.
1189                  */
1190                 if (!((old->flags & new->flags) & IRQF_SHARED) ||
1191                     ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK) ||
1192                     ((old->flags ^ new->flags) & IRQF_ONESHOT))
1193                         goto mismatch;
1194 
1195                 /* All handlers must agree on per-cpuness */
1196                 if ((old->flags & IRQF_PERCPU) !=
1197                     (new->flags & IRQF_PERCPU))
1198                         goto mismatch;
1199 
1200                 /* add new interrupt at end of irq queue */
1201                 do {
1202                         /*
1203                          * Or all existing action->thread_mask bits,
1204                          * so we can find the next zero bit for this
1205                          * new action.
1206                          */
1207                         thread_mask |= old->thread_mask;
1208                         old_ptr = &old->next;
1209                         old = *old_ptr;
1210                 } while (old);
1211                 shared = 1;
1212         }
1213 
1214         /*
1215          * Setup the thread mask for this irqaction for ONESHOT. For
1216          * !ONESHOT irqs the thread mask is 0 so we can avoid a
1217          * conditional in irq_wake_thread().
1218          */
1219         if (new->flags & IRQF_ONESHOT) {
1220                 /*
1221                  * Unlikely to have 32 resp 64 irqs sharing one line,
1222                  * but who knows.
1223                  */
1224                 if (thread_mask == ~0UL) {
1225                         ret = -EBUSY;
1226                         goto out_mask;
1227                 }
1228                 /*
1229                  * The thread_mask for the action is or'ed to
1230                  * desc->thread_active to indicate that the
1231                  * IRQF_ONESHOT thread handler has been woken, but not
1232                  * yet finished. The bit is cleared when a thread
1233                  * completes. When all threads of a shared interrupt
1234                  * line have completed desc->threads_active becomes
1235                  * zero and the interrupt line is unmasked. See
1236                  * handle.c:irq_wake_thread() for further information.
1237                  *
1238                  * If no thread is woken by primary (hard irq context)
1239                  * interrupt handlers, then desc->threads_active is
1240                  * also checked for zero to unmask the irq line in the
1241                  * affected hard irq flow handlers
1242                  * (handle_[fasteoi|level]_irq).
1243                  *
1244                  * The new action gets the first zero bit of
1245                  * thread_mask assigned. See the loop above which or's
1246                  * all existing action->thread_mask bits.
1247                  */
1248                 new->thread_mask = 1 << ffz(thread_mask);
1249 
1250         } else if (new->handler == irq_default_primary_handler &&
1251                    !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1252                 /*
1253                  * The interrupt was requested with handler = NULL, so
1254                  * we use the default primary handler for it. But it
1255                  * does not have the oneshot flag set. In combination
1256                  * with level interrupts this is deadly, because the
1257                  * default primary handler just wakes the thread, then
1258                  * the irq lines is reenabled, but the device still
1259                  * has the level irq asserted. Rinse and repeat....
1260                  *
1261                  * While this works for edge type interrupts, we play
1262                  * it safe and reject unconditionally because we can't
1263                  * say for sure which type this interrupt really
1264                  * has. The type flags are unreliable as the
1265                  * underlying chip implementation can override them.
1266                  */
1267                 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1268                        irq);
1269                 ret = -EINVAL;
1270                 goto out_mask;
1271         }
1272 
1273         if (!shared) {
1274                 ret = irq_request_resources(desc);
1275                 if (ret) {
1276                         pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1277                                new->name, irq, desc->irq_data.chip->name);
1278                         goto out_mask;
1279                 }
1280 
1281                 init_waitqueue_head(&desc->wait_for_threads);
1282 
1283                 /* Setup the type (level, edge polarity) if configured: */
1284                 if (new->flags & IRQF_TRIGGER_MASK) {
1285                         ret = __irq_set_trigger(desc,
1286                                                 new->flags & IRQF_TRIGGER_MASK);
1287 
1288                         if (ret)
1289                                 goto out_mask;
1290                 }
1291 
1292                 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1293                                   IRQS_ONESHOT | IRQS_WAITING);
1294                 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1295 
1296                 if (new->flags & IRQF_PERCPU) {
1297                         irqd_set(&desc->irq_data, IRQD_PER_CPU);
1298                         irq_settings_set_per_cpu(desc);
1299                 }
1300 
1301                 if (new->flags & IRQF_ONESHOT)
1302                         desc->istate |= IRQS_ONESHOT;
1303 
1304                 if (irq_settings_can_autoenable(desc))
1305                         irq_startup(desc, true);
1306                 else
1307                         /* Undo nested disables: */
1308                         desc->depth = 1;
1309 
1310                 /* Exclude IRQ from balancing if requested */
1311                 if (new->flags & IRQF_NOBALANCING) {
1312                         irq_settings_set_no_balancing(desc);
1313                         irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1314                 }
1315 
1316                 /* Set default affinity mask once everything is setup */
1317                 setup_affinity(desc, mask);
1318 
1319         } else if (new->flags & IRQF_TRIGGER_MASK) {
1320                 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1321                 unsigned int omsk = irq_settings_get_trigger_mask(desc);
1322 
1323                 if (nmsk != omsk)
1324                         /* hope the handler works with current  trigger mode */
1325                         pr_warn("irq %d uses trigger mode %u; requested %u\n",
1326                                 irq, nmsk, omsk);
1327         }
1328 
1329         *old_ptr = new;
1330 
1331         irq_pm_install_action(desc, new);
1332 
1333         /* Reset broken irq detection when installing new handler */
1334         desc->irq_count = 0;
1335         desc->irqs_unhandled = 0;
1336 
1337         /*
1338          * Check whether we disabled the irq via the spurious handler
1339          * before. Reenable it and give it another chance.
1340          */
1341         if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1342                 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1343                 __enable_irq(desc);
1344         }
1345 
1346         raw_spin_unlock_irqrestore(&desc->lock, flags);
1347 
1348         /*
1349          * Strictly no need to wake it up, but hung_task complains
1350          * when no hard interrupt wakes the thread up.
1351          */
1352         if (new->thread)
1353                 wake_up_process(new->thread);
1354         if (new->secondary)
1355                 wake_up_process(new->secondary->thread);
1356 
1357         register_irq_proc(irq, desc);
1358         new->dir = NULL;
1359         register_handler_proc(irq, new);
1360         free_cpumask_var(mask);
1361 
1362         return 0;
1363 
1364 mismatch:
1365         if (!(new->flags & IRQF_PROBE_SHARED)) {
1366                 pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1367                        irq, new->flags, new->name, old->flags, old->name);
1368 #ifdef CONFIG_DEBUG_SHIRQ
1369                 dump_stack();
1370 #endif
1371         }
1372         ret = -EBUSY;
1373 
1374 out_mask:
1375         raw_spin_unlock_irqrestore(&desc->lock, flags);
1376         free_cpumask_var(mask);
1377 
1378 out_thread:
1379         if (new->thread) {
1380                 struct task_struct *t = new->thread;
1381 
1382                 new->thread = NULL;
1383                 kthread_stop(t);
1384                 put_task_struct(t);
1385         }
1386         if (new->secondary && new->secondary->thread) {
1387                 struct task_struct *t = new->secondary->thread;
1388 
1389                 new->secondary->thread = NULL;
1390                 kthread_stop(t);
1391                 put_task_struct(t);
1392         }
1393 out_mput:
1394         module_put(desc->owner);
1395         return ret;
1396 }
1397 
1398 /**
1399  *      setup_irq - setup an interrupt
1400  *      @irq: Interrupt line to setup
1401  *      @act: irqaction for the interrupt
1402  *
1403  * Used to statically setup interrupts in the early boot process.
1404  */
1405 int setup_irq(unsigned int irq, struct irqaction *act)
1406 {
1407         int retval;
1408         struct irq_desc *desc = irq_to_desc(irq);
1409 
1410         if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1411                 return -EINVAL;
1412         chip_bus_lock(desc);
1413         retval = __setup_irq(irq, desc, act);
1414         chip_bus_sync_unlock(desc);
1415 
1416         return retval;
1417 }
1418 EXPORT_SYMBOL_GPL(setup_irq);
1419 
1420 /*
1421  * Internal function to unregister an irqaction - used to free
1422  * regular and special interrupts that are part of the architecture.
1423  */
1424 static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
1425 {
1426         struct irq_desc *desc = irq_to_desc(irq);
1427         struct irqaction *action, **action_ptr;
1428         unsigned long flags;
1429 
1430         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1431 
1432         if (!desc)
1433                 return NULL;
1434 
1435         chip_bus_lock(desc);
1436         raw_spin_lock_irqsave(&desc->lock, flags);
1437 
1438         /*
1439          * There can be multiple actions per IRQ descriptor, find the right
1440          * one based on the dev_id:
1441          */
1442         action_ptr = &desc->action;
1443         for (;;) {
1444                 action = *action_ptr;
1445 
1446                 if (!action) {
1447                         WARN(1, "Trying to free already-free IRQ %d\n", irq);
1448                         raw_spin_unlock_irqrestore(&desc->lock, flags);
1449                         chip_bus_sync_unlock(desc);
1450                         return NULL;
1451                 }
1452 
1453                 if (action->dev_id == dev_id)
1454                         break;
1455                 action_ptr = &action->next;
1456         }
1457 
1458         /* Found it - now remove it from the list of entries: */
1459         *action_ptr = action->next;
1460 
1461         irq_pm_remove_action(desc, action);
1462 
1463         /* If this was the last handler, shut down the IRQ line: */
1464         if (!desc->action) {
1465                 irq_settings_clr_disable_unlazy(desc);
1466                 irq_shutdown(desc);
1467                 irq_release_resources(desc);
1468         }
1469 
1470 #ifdef CONFIG_SMP
1471         /* make sure affinity_hint is cleaned up */
1472         if (WARN_ON_ONCE(desc->affinity_hint))
1473                 desc->affinity_hint = NULL;
1474 #endif
1475 
1476         raw_spin_unlock_irqrestore(&desc->lock, flags);
1477         chip_bus_sync_unlock(desc);
1478 
1479         unregister_handler_proc(irq, action);
1480 
1481         /* Make sure it's not being used on another CPU: */
1482         synchronize_irq(irq);
1483 
1484 #ifdef CONFIG_DEBUG_SHIRQ
1485         /*
1486          * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1487          * event to happen even now it's being freed, so let's make sure that
1488          * is so by doing an extra call to the handler ....
1489          *
1490          * ( We do this after actually deregistering it, to make sure that a
1491          *   'real' IRQ doesn't run in * parallel with our fake. )
1492          */
1493         if (action->flags & IRQF_SHARED) {
1494                 local_irq_save(flags);
1495                 action->handler(irq, dev_id);
1496                 local_irq_restore(flags);
1497         }
1498 #endif
1499 
1500         if (action->thread) {
1501                 kthread_stop(action->thread);
1502                 put_task_struct(action->thread);
1503                 if (action->secondary && action->secondary->thread) {
1504                         kthread_stop(action->secondary->thread);
1505                         put_task_struct(action->secondary->thread);
1506                 }
1507         }
1508 
1509         module_put(desc->owner);
1510         kfree(action->secondary);
1511         return action;
1512 }
1513 
1514 /**
1515  *      remove_irq - free an interrupt
1516  *      @irq: Interrupt line to free
1517  *      @act: irqaction for the interrupt
1518  *
1519  * Used to remove interrupts statically setup by the early boot process.
1520  */
1521 void remove_irq(unsigned int irq, struct irqaction *act)
1522 {
1523         struct irq_desc *desc = irq_to_desc(irq);
1524 
1525         if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1526             __free_irq(irq, act->dev_id);
1527 }
1528 EXPORT_SYMBOL_GPL(remove_irq);
1529 
1530 /**
1531  *      free_irq - free an interrupt allocated with request_irq
1532  *      @irq: Interrupt line to free
1533  *      @dev_id: Device identity to free
1534  *
1535  *      Remove an interrupt handler. The handler is removed and if the
1536  *      interrupt line is no longer in use by any driver it is disabled.
1537  *      On a shared IRQ the caller must ensure the interrupt is disabled
1538  *      on the card it drives before calling this function. The function
1539  *      does not return until any executing interrupts for this IRQ
1540  *      have completed.
1541  *
1542  *      This function must not be called from interrupt context.
1543  */
1544 void free_irq(unsigned int irq, void *dev_id)
1545 {
1546         struct irq_desc *desc = irq_to_desc(irq);
1547 
1548         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1549                 return;
1550 
1551 #ifdef CONFIG_SMP
1552         if (WARN_ON(desc->affinity_notify))
1553                 desc->affinity_notify = NULL;
1554 #endif
1555 
1556         kfree(__free_irq(irq, dev_id));
1557 }
1558 EXPORT_SYMBOL(free_irq);
1559 
1560 /**
1561  *      request_threaded_irq - allocate an interrupt line
1562  *      @irq: Interrupt line to allocate
1563  *      @handler: Function to be called when the IRQ occurs.
1564  *                Primary handler for threaded interrupts
1565  *                If NULL and thread_fn != NULL the default
1566  *                primary handler is installed
1567  *      @thread_fn: Function called from the irq handler thread
1568  *                  If NULL, no irq thread is created
1569  *      @irqflags: Interrupt type flags
1570  *      @devname: An ascii name for the claiming device
1571  *      @dev_id: A cookie passed back to the handler function
1572  *
1573  *      This call allocates interrupt resources and enables the
1574  *      interrupt line and IRQ handling. From the point this
1575  *      call is made your handler function may be invoked. Since
1576  *      your handler function must clear any interrupt the board
1577  *      raises, you must take care both to initialise your hardware
1578  *      and to set up the interrupt handler in the right order.
1579  *
1580  *      If you want to set up a threaded irq handler for your device
1581  *      then you need to supply @handler and @thread_fn. @handler is
1582  *      still called in hard interrupt context and has to check
1583  *      whether the interrupt originates from the device. If yes it
1584  *      needs to disable the interrupt on the device and return
1585  *      IRQ_WAKE_THREAD which will wake up the handler thread and run
1586  *      @thread_fn. This split handler design is necessary to support
1587  *      shared interrupts.
1588  *
1589  *      Dev_id must be globally unique. Normally the address of the
1590  *      device data structure is used as the cookie. Since the handler
1591  *      receives this value it makes sense to use it.
1592  *
1593  *      If your interrupt is shared you must pass a non NULL dev_id
1594  *      as this is required when freeing the interrupt.
1595  *
1596  *      Flags:
1597  *
1598  *      IRQF_SHARED             Interrupt is shared
1599  *      IRQF_TRIGGER_*          Specify active edge(s) or level
1600  *
1601  */
1602 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
1603                          irq_handler_t thread_fn, unsigned long irqflags,
1604                          const char *devname, void *dev_id)
1605 {
1606         struct irqaction *action;
1607         struct irq_desc *desc;
1608         int retval;
1609 
1610         if (irq == IRQ_NOTCONNECTED)
1611                 return -ENOTCONN;
1612 
1613         /*
1614          * Sanity-check: shared interrupts must pass in a real dev-ID,
1615          * otherwise we'll have trouble later trying to figure out
1616          * which interrupt is which (messes up the interrupt freeing
1617          * logic etc).
1618          *
1619          * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
1620          * it cannot be set along with IRQF_NO_SUSPEND.
1621          */
1622         if (((irqflags & IRQF_SHARED) && !dev_id) ||
1623             (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
1624             ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
1625                 return -EINVAL;
1626 
1627         desc = irq_to_desc(irq);
1628         if (!desc)
1629                 return -EINVAL;
1630 
1631         if (!irq_settings_can_request(desc) ||
1632             WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1633                 return -EINVAL;
1634 
1635         if (!handler) {
1636                 if (!thread_fn)
1637                         return -EINVAL;
1638                 handler = irq_default_primary_handler;
1639         }
1640 
1641         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1642         if (!action)
1643                 return -ENOMEM;
1644 
1645         action->handler = handler;
1646         action->thread_fn = thread_fn;
1647         action->flags = irqflags;
1648         action->name = devname;
1649         action->dev_id = dev_id;
1650 
1651         chip_bus_lock(desc);
1652         retval = __setup_irq(irq, desc, action);
1653         chip_bus_sync_unlock(desc);
1654 
1655         if (retval) {
1656                 kfree(action->secondary);
1657                 kfree(action);
1658         }
1659 
1660 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
1661         if (!retval && (irqflags & IRQF_SHARED)) {
1662                 /*
1663                  * It's a shared IRQ -- the driver ought to be prepared for it
1664                  * to happen immediately, so let's make sure....
1665                  * We disable the irq to make sure that a 'real' IRQ doesn't
1666                  * run in parallel with our fake.
1667                  */
1668                 unsigned long flags;
1669 
1670                 disable_irq(irq);
1671                 local_irq_save(flags);
1672 
1673                 handler(irq, dev_id);
1674 
1675                 local_irq_restore(flags);
1676                 enable_irq(irq);
1677         }
1678 #endif
1679         return retval;
1680 }
1681 EXPORT_SYMBOL(request_threaded_irq);
1682 
1683 /**
1684  *      request_any_context_irq - allocate an interrupt line
1685  *      @irq: Interrupt line to allocate
1686  *      @handler: Function to be called when the IRQ occurs.
1687  *                Threaded handler for threaded interrupts.
1688  *      @flags: Interrupt type flags
1689  *      @name: An ascii name for the claiming device
1690  *      @dev_id: A cookie passed back to the handler function
1691  *
1692  *      This call allocates interrupt resources and enables the
1693  *      interrupt line and IRQ handling. It selects either a
1694  *      hardirq or threaded handling method depending on the
1695  *      context.
1696  *
1697  *      On failure, it returns a negative value. On success,
1698  *      it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
1699  */
1700 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
1701                             unsigned long flags, const char *name, void *dev_id)
1702 {
1703         struct irq_desc *desc;
1704         int ret;
1705 
1706         if (irq == IRQ_NOTCONNECTED)
1707                 return -ENOTCONN;
1708 
1709         desc = irq_to_desc(irq);
1710         if (!desc)
1711                 return -EINVAL;
1712 
1713         if (irq_settings_is_nested_thread(desc)) {
1714                 ret = request_threaded_irq(irq, NULL, handler,
1715                                            flags, name, dev_id);
1716                 return !ret ? IRQC_IS_NESTED : ret;
1717         }
1718 
1719         ret = request_irq(irq, handler, flags, name, dev_id);
1720         return !ret ? IRQC_IS_HARDIRQ : ret;
1721 }
1722 EXPORT_SYMBOL_GPL(request_any_context_irq);
1723 
1724 void enable_percpu_irq(unsigned int irq, unsigned int type)
1725 {
1726         unsigned int cpu = smp_processor_id();
1727         unsigned long flags;
1728         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1729 
1730         if (!desc)
1731                 return;
1732 
1733         type &= IRQ_TYPE_SENSE_MASK;
1734         if (type != IRQ_TYPE_NONE) {
1735                 int ret;
1736 
1737                 ret = __irq_set_trigger(desc, type);
1738 
1739                 if (ret) {
1740                         WARN(1, "failed to set type for IRQ%d\n", irq);
1741                         goto out;
1742                 }
1743         }
1744 
1745         irq_percpu_enable(desc, cpu);
1746 out:
1747         irq_put_desc_unlock(desc, flags);
1748 }
1749 EXPORT_SYMBOL_GPL(enable_percpu_irq);
1750 
1751 /**
1752  * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
1753  * @irq:        Linux irq number to check for
1754  *
1755  * Must be called from a non migratable context. Returns the enable
1756  * state of a per cpu interrupt on the current cpu.
1757  */
1758 bool irq_percpu_is_enabled(unsigned int irq)
1759 {
1760         unsigned int cpu = smp_processor_id();
1761         struct irq_desc *desc;
1762         unsigned long flags;
1763         bool is_enabled;
1764 
1765         desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1766         if (!desc)
1767                 return false;
1768 
1769         is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
1770         irq_put_desc_unlock(desc, flags);
1771 
1772         return is_enabled;
1773 }
1774 EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
1775 
1776 void disable_percpu_irq(unsigned int irq)
1777 {
1778         unsigned int cpu = smp_processor_id();
1779         unsigned long flags;
1780         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1781 
1782         if (!desc)
1783                 return;
1784 
1785         irq_percpu_disable(desc, cpu);
1786         irq_put_desc_unlock(desc, flags);
1787 }
1788 EXPORT_SYMBOL_GPL(disable_percpu_irq);
1789 
1790 /*
1791  * Internal function to unregister a percpu irqaction.
1792  */
1793 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1794 {
1795         struct irq_desc *desc = irq_to_desc(irq);
1796         struct irqaction *action;
1797         unsigned long flags;
1798 
1799         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1800 
1801         if (!desc)
1802                 return NULL;
1803 
1804         raw_spin_lock_irqsave(&desc->lock, flags);
1805 
1806         action = desc->action;
1807         if (!action || action->percpu_dev_id != dev_id) {
1808                 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1809                 goto bad;
1810         }
1811 
1812         if (!cpumask_empty(desc->percpu_enabled)) {
1813                 WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
1814                      irq, cpumask_first(desc->percpu_enabled));
1815                 goto bad;
1816         }
1817 
1818         /* Found it - now remove it from the list of entries: */
1819         desc->action = NULL;
1820 
1821         raw_spin_unlock_irqrestore(&desc->lock, flags);
1822 
1823         unregister_handler_proc(irq, action);
1824 
1825         module_put(desc->owner);
1826         return action;
1827 
1828 bad:
1829         raw_spin_unlock_irqrestore(&desc->lock, flags);
1830         return NULL;
1831 }
1832 
1833 /**
1834  *      remove_percpu_irq - free a per-cpu interrupt
1835  *      @irq: Interrupt line to free
1836  *      @act: irqaction for the interrupt
1837  *
1838  * Used to remove interrupts statically setup by the early boot process.
1839  */
1840 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
1841 {
1842         struct irq_desc *desc = irq_to_desc(irq);
1843 
1844         if (desc && irq_settings_is_per_cpu_devid(desc))
1845             __free_percpu_irq(irq, act->percpu_dev_id);
1846 }
1847 
1848 /**
1849  *      free_percpu_irq - free an interrupt allocated with request_percpu_irq
1850  *      @irq: Interrupt line to free
1851  *      @dev_id: Device identity to free
1852  *
1853  *      Remove a percpu interrupt handler. The handler is removed, but
1854  *      the interrupt line is not disabled. This must be done on each
1855  *      CPU before calling this function. The function does not return
1856  *      until any executing interrupts for this IRQ have completed.
1857  *
1858  *      This function must not be called from interrupt context.
1859  */
1860 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1861 {
1862         struct irq_desc *desc = irq_to_desc(irq);
1863 
1864         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1865                 return;
1866 
1867         chip_bus_lock(desc);
1868         kfree(__free_percpu_irq(irq, dev_id));
1869         chip_bus_sync_unlock(desc);
1870 }
1871 EXPORT_SYMBOL_GPL(free_percpu_irq);
1872 
1873 /**
1874  *      setup_percpu_irq - setup a per-cpu interrupt
1875  *      @irq: Interrupt line to setup
1876  *      @act: irqaction for the interrupt
1877  *
1878  * Used to statically setup per-cpu interrupts in the early boot process.
1879  */
1880 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
1881 {
1882         struct irq_desc *desc = irq_to_desc(irq);
1883         int retval;
1884 
1885         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1886                 return -EINVAL;
1887         chip_bus_lock(desc);
1888         retval = __setup_irq(irq, desc, act);
1889         chip_bus_sync_unlock(desc);
1890 
1891         return retval;
1892 }
1893 
1894 /**
1895  *      request_percpu_irq - allocate a percpu interrupt line
1896  *      @irq: Interrupt line to allocate
1897  *      @handler: Function to be called when the IRQ occurs.
1898  *      @devname: An ascii name for the claiming device
1899  *      @dev_id: A percpu cookie passed back to the handler function
1900  *
1901  *      This call allocates interrupt resources and enables the
1902  *      interrupt on the local CPU. If the interrupt is supposed to be
1903  *      enabled on other CPUs, it has to be done on each CPU using
1904  *      enable_percpu_irq().
1905  *
1906  *      Dev_id must be globally unique. It is a per-cpu variable, and
1907  *      the handler gets called with the interrupted CPU's instance of
1908  *      that variable.
1909  */
1910 int request_percpu_irq(unsigned int irq, irq_handler_t handler,
1911                        const char *devname, void __percpu *dev_id)
1912 {
1913         struct irqaction *action;
1914         struct irq_desc *desc;
1915         int retval;
1916 
1917         if (!dev_id)
1918                 return -EINVAL;
1919 
1920         desc = irq_to_desc(irq);
1921         if (!desc || !irq_settings_can_request(desc) ||
1922             !irq_settings_is_per_cpu_devid(desc))
1923                 return -EINVAL;
1924 
1925         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1926         if (!action)
1927                 return -ENOMEM;
1928 
1929         action->handler = handler;
1930         action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND;
1931         action->name = devname;
1932         action->percpu_dev_id = dev_id;
1933 
1934         chip_bus_lock(desc);
1935         retval = __setup_irq(irq, desc, action);
1936         chip_bus_sync_unlock(desc);
1937 
1938         if (retval)
1939                 kfree(action);
1940 
1941         return retval;
1942 }
1943 EXPORT_SYMBOL_GPL(request_percpu_irq);
1944 
1945 /**
1946  *      irq_get_irqchip_state - returns the irqchip state of a interrupt.
1947  *      @irq: Interrupt line that is forwarded to a VM
1948  *      @which: One of IRQCHIP_STATE_* the caller wants to know about
1949  *      @state: a pointer to a boolean where the state is to be storeed
1950  *
1951  *      This call snapshots the internal irqchip state of an
1952  *      interrupt, returning into @state the bit corresponding to
1953  *      stage @which
1954  *
1955  *      This function should be called with preemption disabled if the
1956  *      interrupt controller has per-cpu registers.
1957  */
1958 int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
1959                           bool *state)
1960 {
1961         struct irq_desc *desc;
1962         struct irq_data *data;
1963         struct irq_chip *chip;
1964         unsigned long flags;
1965         int err = -EINVAL;
1966 
1967         desc = irq_get_desc_buslock(irq, &flags, 0);
1968         if (!desc)
1969                 return err;
1970 
1971         data = irq_desc_get_irq_data(desc);
1972 
1973         do {
1974                 chip = irq_data_get_irq_chip(data);
1975                 if (chip->irq_get_irqchip_state)
1976                         break;
1977 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1978                 data = data->parent_data;
1979 #else
1980                 data = NULL;
1981 #endif
1982         } while (data);
1983 
1984         if (data)
1985                 err = chip->irq_get_irqchip_state(data, which, state);
1986 
1987         irq_put_desc_busunlock(desc, flags);
1988         return err;
1989 }
1990 EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
1991 
1992 /**
1993  *      irq_set_irqchip_state - set the state of a forwarded interrupt.
1994  *      @irq: Interrupt line that is forwarded to a VM
1995  *      @which: State to be restored (one of IRQCHIP_STATE_*)
1996  *      @val: Value corresponding to @which
1997  *
1998  *      This call sets the internal irqchip state of an interrupt,
1999  *      depending on the value of @which.
2000  *
2001  *      This function should be called with preemption disabled if the
2002  *      interrupt controller has per-cpu registers.
2003  */
2004 int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2005                           bool val)
2006 {
2007         struct irq_desc *desc;
2008         struct irq_data *data;
2009         struct irq_chip *chip;
2010         unsigned long flags;
2011         int err = -EINVAL;
2012 
2013         desc = irq_get_desc_buslock(irq, &flags, 0);
2014         if (!desc)
2015                 return err;
2016 
2017         data = irq_desc_get_irq_data(desc);
2018 
2019         do {
2020                 chip = irq_data_get_irq_chip(data);
2021                 if (chip->irq_set_irqchip_state)
2022                         break;
2023 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2024                 data = data->parent_data;
2025 #else
2026                 data = NULL;
2027 #endif
2028         } while (data);
2029 
2030         if (data)
2031                 err = chip->irq_set_irqchip_state(data, which, val);
2032 
2033         irq_put_desc_busunlock(desc, flags);
2034         return err;
2035 }
2036 EXPORT_SYMBOL_GPL(irq_set_irqchip_state);
2037 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us