nuttx/sched/irq/irq_csection.c

/****************************************************************************
 * sched/irq/irq_csection.c
 *
 *   Copyright (C) 2016-2018 Gregory Nutt. All rights reserved.
 *   Author: Gregory Nutt <gnutt@nuttx.org>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name NuttX nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

/****************************************************************************
 * Included Files
 ****************************************************************************/

#include <nuttx/config.h>

#include <sys/types.h>

#include <nuttx/init.h>
#include <nuttx/spinlock.h>
#include <nuttx/sched_note.h>
#include <arch/irq.h>

#include "sched/sched.h"
#include "irq/irq.h"

#ifdef CONFIG_IRQCOUNT

/****************************************************************************
 * Public Data
 ****************************************************************************/

#ifdef CONFIG_SMP
/* This is the spinlock that enforces critical sections when interrupts are
 * disabled.
 */

volatile spinlock_t g_cpu_irqlock SP_SECTION = SP_UNLOCKED;

/* Used to keep track of which CPU(s) hold the IRQ lock. */

volatile spinlock_t g_cpu_irqsetlock SP_SECTION;
volatile cpu_set_t g_cpu_irqset SP_SECTION;

/* Handles nested calls to enter_critical section from interrupt handlers */

volatile uint8_t g_cpu_nestcount[CONFIG_SMP_NCPUS];
#endif

/****************************************************************************
 * Private Functions
 ****************************************************************************/

/****************************************************************************
 * Name: irq_waitlock
 *
 * Description:
 *   Spin to get g_irq_waitlock, handling a known deadlock condition:
 *
 *   A deadlock may occur if enter_critical_section is called from an
 *   interrupt handler.  Suppose:
 *
 *   - CPUn is in a critical section and has the g_cpu_irqlock spinlock.
 *   - CPUm takes an interrupt and attempts to enter the critical section.
 *   - It spins waiting on g_cpu_irqlock with interrupts disabled.
 *   - CPUn calls up_cpu_pause() to pause operation on CPUm.  This will
 *     issue an inter-CPU interrupt to CPUm
 *   - But interrupts are disabled on CPUm so the up_cpu_pause() is never
 *     handled, causing the deadlock.
 *
 *   This same deadlock can occur in the normal tasking case:
 *
 *   - A task on CPUn enters a critical section and has the g_cpu_irqlock
 *     spinlock.
 *   - Another task on CPUm attempts to enter the critical section but has
 *     to wait, spinning to get g_cpu_irqlock with interrupts disabled.
 *   - The task on CPUn causes a new task to become ready-to-run and the
 *     scheduler selects CPUm.  CPUm is requested to pause via a pause
 *     interrupt.
 *   - But the task on CPUm is also attempting to enter the critical
 *     section.  Since it is spinning with interrupts disabled, CPUm cannot
 *     process the pending pause interrupt, causing the deadlock.
 *
 *   This function detects this deadlock condition while spinning with \
 *   interrupts disabled.
 *
 * Input Parameters:
 *   cpu - The index of CPU that is trying to enter the critical section.
 *
 * Returned Value:
 *   True:  The g_cpu_irqlock spinlock has been taken.
 *   False: The g_cpu_irqlock spinlock has not been taken yet, but there is
 *          a pending pause interrupt request.
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
static inline bool irq_waitlock(int cpu)
{
#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS
  FAR struct tcb_s *tcb = current_task(cpu);

  /* Notify that we are waiting for a spinlock */

  sched_note_spinlock(tcb, &g_cpu_irqlock);
#endif

  /* Duplicate the spin_lock() logic from spinlock.c, but adding the check
   * for the deadlock condition.
   */

  while (spin_trylock(&g_cpu_irqlock) == SP_LOCKED)
    {
      /* Is a pause request pending? */

      if (up_cpu_pausereq(cpu))
        {
          /* Yes.. some other CPU is requesting to pause this CPU!
           * Abort the wait and return false.
           */

#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS
          /* Notify that we are waiting for a spinlock */

          sched_note_spinabort(tcb, &g_cpu_irqlock);
#endif

          return false;
        }

      SP_DSB();
    }

  /* We have g_cpu_irqlock! */

#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS
  /* Notify that we have the spinlock */

  sched_note_spinlocked(tcb, &g_cpu_irqlock);
#endif

  SP_DMB();
  return true;
}
#endif

/****************************************************************************
 * Public Functions
 ****************************************************************************/

/****************************************************************************
 * Name: enter_critical_section
 *
 * Description:
 *   Take the CPU IRQ lock and disable interrupts on all CPUs.  A thread-
 *   specific counter is increment to indicate that the thread has IRQs
 *   disabled and to support nested calls to enter_critical_section().
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
irqstate_t enter_critical_section(void)
{
  FAR struct tcb_s *rtcb;
  irqstate_t ret;
  int cpu;

  /* Disable interrupts.
   *
   * NOTE 1: Ideally this should disable interrupts on all CPUs, but most
   * architectures only support disabling interrupts on the local CPU.
   * NOTE 2: Interrupts may already be disabled, but we call up_irq_save()
   * unconditionally because we need to return valid interrupt status in any
   * event.
   * NOTE 3: We disable local interrupts BEFORE taking the spinlock in order
   * to prevent possible waits on the spinlock from interrupt handling on
   * the local CPU.
   */

try_again:

  ret = up_irq_save();

  /* Verify that the system has sufficiently initialized so that the task
   * lists are valid.
   */

  if (g_os_initstate >= OSINIT_TASKLISTS)
    {
      /* If called from an interrupt handler, then just take the spinlock.
       * If we are already in a critical section, this will lock the CPU
       * in the interrupt handler.  Sounds worse than it is.
       */

      if (up_interrupt_context())
        {
          /* We are in an interrupt handler.  How can this happen?
           *
           *   1. We were not in a critical section when the interrupt
           *      occurred.  In this case, the interrupt was entered with:
           *
           *      g_cpu_irqlock = SP_UNLOCKED.
           *      g_cpu_nestcount = 0
           *      All CPU bits in g_cpu_irqset should be zero
           *
           *   2. We were in a critical section and interrupts on this
           *      this CPU were disabled -- this is an impossible case.
           *
           *   3. We were in critical section, but up_irq_save() only
           *      disabled local interrupts on a different CPU;
           *      Interrupts could still be enabled on this CPU.
           *
           *      g_cpu_irqlock = SP_LOCKED.
           *      g_cpu_nestcount = 0
           *      The bit in g_cpu_irqset for this CPU should be zero
           *
           *   4. An extension of 3 is that we may be re-entered numerous
           *      times from the same interrupt handler.  In that case:
           *
           *      g_cpu_irqlock = SP_LOCKED.
           *      g_cpu_nestcount > 0
           *      The bit in g_cpu_irqset for this CPU should be zero
           *
           * NOTE: However, the interrupt entry conditions can change due
           * to previous processing by the interrupt handler that may
           * instantiate a new thread that has irqcount > 0 and may then
           * set the bit in g_cpu_irqset and g_cpu_irqlock = SP_LOCKED
           */

          /* Handle nested calls to enter_critical_section() from the same
           * interrupt.
           */

          cpu = this_cpu();
          if (g_cpu_nestcount[cpu] > 0)
            {
              DEBUGASSERT(spin_islocked(&g_cpu_irqlock) &&
                          g_cpu_nestcount[cpu] < UINT8_MAX);
              g_cpu_nestcount[cpu]++;
            }

          /* This is the first call to enter_critical_section from the
           * interrupt handler.
           */

          else
            {
              /* Make sure that the g_cpu_irqlock() was not already set
               * by previous logic on this CPU that was executed by the
               * interrupt handler.  We know that the bit in g_cpu_irqset
               * for this CPU was zero on entry into the interrupt handler,
               * so if it is non-zero now then we know that was the case.
               */

              if ((g_cpu_irqset & (1 << cpu)) == 0)
                {
                  /* Wait until we can get the spinlock (meaning that we are
                   * no longer blocked by the critical section).
                   */

                  if (!irq_waitlock(cpu))
                    {
                      /* We are in a deadlock condition due to a pending
                       * pause request interrupt request.  Break the
                       * deadlock by handling the pause interrupt now.
                       */

                      DEBUGVERIFY(up_cpu_paused(cpu));
                    }
                }

              /* In any event, the nesting count is now one */

              g_cpu_nestcount[cpu] = 1;

              /* Also set the CPU bit so that other CPUs will be aware that this
               * CPU holds the critical section.
               */

              spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock,
                          &g_cpu_irqlock);
            }
        }
      else
        {
          /* Normal tasking environment. */
          /* Get the TCB of the currently executing task on this CPU (avoid
           * using this_task() which can recurse.
           */

          cpu  = this_cpu();
          rtcb = current_task(cpu);
          DEBUGASSERT(rtcb != NULL);

          /* Do we already have interrupts disabled? */

          if (rtcb->irqcount > 0)
            {
              /* Yes... make sure that the spinlock is set and increment the
               * IRQ lock count.
               *
               * NOTE: If irqcount > 0 then (1) we are in a critical section, and
               * (2) this CPU should hold the lock.
               */

              DEBUGASSERT(spin_islocked(&g_cpu_irqlock) &&
                          (g_cpu_irqset & (1 << this_cpu())) != 0 &&
                          rtcb->irqcount < INT16_MAX);
              rtcb->irqcount++;
            }
          else
            {
              /* If we get here with irqcount == 0, then we know that the
               * current task running on this CPU is not in a critical
               * section.  However other tasks on other CPUs may be in a
               * critical section.  If so, we must wait until they release
               * the spinlock.
               */

              DEBUGASSERT((g_cpu_irqset & (1 << cpu)) == 0);

              if (!irq_waitlock(cpu))
                {
                  /* We are in a deadlock condition due to a pending pause
                   * request interrupt.  Re-enable interrupts on this CPU
                   * and try again.  Briefly re-enabling interrupts should
                   * be sufficient to permit processing the pending pause
                   * request.
                   *
                   * NOTE: This should never happen on architectures like
                   * the Cortex-A; the inter-CPU interrupt (SGI) is not
                   * maskable.
                   */

                  up_irq_restore(ret);
                  goto try_again;
                }

              /* The set the lock count to 1.
               *
               * Interrupts disables must follow a stacked order.  We
               * cannot other context switches to re-order the enabling
               * disabling of interrupts.
               *
               * The scheduler accomplishes this by treating the irqcount
               * like lockcount:  Both will disable pre-emption.
               */

              spin_setbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock,
                          &g_cpu_irqlock);
              rtcb->irqcount = 1;

              /* Note that we have entered the critical section */

#ifdef CONFIG_SCHED_CRITMONITOR
              sched_critmon_csection(rtcb, true);
#endif
#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
              sched_note_csection(rtcb, true);
#endif
            }
        }
    }

  /* Return interrupt status */

  return ret;
}

#else

irqstate_t enter_critical_section(void)
{
  irqstate_t ret;

  /* Disable interrupts */

  ret = up_irq_save();

  /* Check if we were called from an interrupt handler and that the task
   * lists have been initialized.
   */

  if (!up_interrupt_context() && g_os_initstate >= OSINIT_TASKLISTS)
    {
      FAR struct tcb_s *rtcb = this_task();
      DEBUGASSERT(rtcb != NULL);

      /* Have we just entered the critical section?  Or is this a nested
       * call to enter_critical_section.
       */

      DEBUGASSERT(rtcb->irqcount >= 0 && rtcb->irqcount < UINT16_MAX);
      if (++rtcb->irqcount == 1)
        {
          /* Note that we have entered the critical section */

#ifdef CONFIG_SCHED_CRITMONITOR
          sched_critmon_csection(rtcb, true);
#endif
#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
          sched_note_csection(rtcb, true);
#endif
        }
    }

  /* Return interrupt status */

  return ret;
}
#endif

/****************************************************************************
 * Name: leave_critical_section
 *
 * Description:
 *   Decrement the IRQ lock count and if it decrements to zero then release
 *   the spinlock.
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
void leave_critical_section(irqstate_t flags)
{
  int cpu;

  /* Verify that the system has sufficiently initialized so that the task
   * lists are valid.
   */

  if (g_os_initstate >= OSINIT_TASKLISTS)
    {
      /* If called from an interrupt handler, then just release the
       * spinlock.  The interrupt handling logic should already hold the
       * spinlock if enter_critical_section() has been called.  Unlocking
       * the spinlock will allow interrupt handlers on other CPUs to execute
       * again.
       */

      if (up_interrupt_context())
        {
          /* We are in an interrupt handler. Check if the last call to
           * enter_critical_section() was nested.
           */

          cpu = this_cpu();
          if (g_cpu_nestcount[cpu] > 1)
            {
              /* Yes.. then just decrement the nesting count */

              DEBUGASSERT(spin_islocked(&g_cpu_irqlock));
              g_cpu_nestcount[cpu]--;
            }
          else
            {
              /* No, not nested. Restore the g_cpu_irqset for this CPU
               * and release the spinlock (if necessary).
               */

              DEBUGASSERT(spin_islocked(&g_cpu_irqlock) &&
                          g_cpu_nestcount[cpu] == 1);

              FAR struct tcb_s *rtcb = current_task(cpu);
              DEBUGASSERT(rtcb != NULL);

              if (rtcb->irqcount <= 0)
                {
                  spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock,
                              &g_cpu_irqlock);
                }

              g_cpu_nestcount[cpu] = 0;
            }
        }
      else
        {
          FAR struct tcb_s *rtcb;

          /* Get the TCB of the currently executing task on this CPU (avoid
           * using this_task() which can recurse.
           */

          cpu  = this_cpu();
          rtcb = current_task(cpu);
          DEBUGASSERT(rtcb != NULL && rtcb->irqcount > 0);

          /* Normal tasking context.  We need to coordinate with other
           * tasks.
           *
           * Will we still have interrupts disabled after decrementing the
           * count?
           */

          if (rtcb->irqcount > 1)
            {
              /* Yes... the spinlock should remain set */

              DEBUGASSERT(spin_islocked(&g_cpu_irqlock));
              rtcb->irqcount--;
            }
          else
            {
              /* No.. Note that we have left the critical section */

#ifdef CONFIG_SCHED_CRITMONITOR
              sched_critmon_csection(rtcb, false);
#endif
#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
              sched_note_csection(rtcb, false);
#endif
              /* Decrement our count on the lock.  If all CPUs have
               * released, then unlock the spinlock.
               */

              DEBUGASSERT(spin_islocked(&g_cpu_irqlock) &&
                          (g_cpu_irqset & (1 << cpu)) != 0);

              /* Check if releasing the lock held by this CPU will unlock the
               * critical section.
               */

              if ((g_cpu_irqset & ~(1 << cpu)) == 0)
                {
                  /* Yes.. Check if there are pending tasks and that pre-emption
                   * is also enabled.  This is necessary because we may have
                   * deferred the up_release_pending() call in sched_unlock()
                   * because we were within a critical section then.
                   */

                  if (g_pendingtasks.head != NULL && !sched_islocked_global())
                    {
                      /* Release any ready-to-run tasks that have collected
                       * in g_pendingtasks.  NOTE: This operation has a very
                       * high likelihood of causing this task to be switched
                       * out!
                       */

                      up_release_pending();
                    }
                }

              /* Now, possibly on return from a context switch, clear our
               * count on the lock.  If all CPUs have released the lock,
               * then unlock the global IRQ spinlock.
               */

              rtcb->irqcount = 0;
              spin_clrbit(&g_cpu_irqset, cpu, &g_cpu_irqsetlock,
                          &g_cpu_irqlock);

              /* Have all CPUs released the lock? */
            }
        }
    }

  /* Restore the previous interrupt state which may still be interrupts
   * disabled (but we don't have a mechanism to verify that now)
   */

  up_irq_restore(flags);
}

#else

void leave_critical_section(irqstate_t flags)
{
  /* Check if we were called from an interrupt handler and that the tasks
   * lists have been initialized.
   */

  if (!up_interrupt_context() && g_os_initstate >= OSINIT_TASKLISTS)
    {
      FAR struct tcb_s *rtcb = this_task();
      DEBUGASSERT(rtcb != NULL);

      /* Have we left entered the critical section?  Or are we still nested. */

      DEBUGASSERT(rtcb->irqcount > 0);
      if (--rtcb->irqcount <= 0)
        {
          /* Note that we have left the critical section */

#ifdef CONFIG_SCHED_CRITMONITOR
          sched_critmon_csection(rtcb, false);
#endif
#ifdef CONFIG_SCHED_INSTRUMENTATION_CSECTION
          sched_note_csection(rtcb, false);
#endif
        }
    }

  /* Restore the previous interrupt state. */

  up_irq_restore(flags);
}
#endif

/****************************************************************************
 * Name:  irq_cpu_locked
 *
 * Description:
 *   Test if the IRQ lock set OR if this CPU holds the IRQ lock
 *   There is an interaction with pre-emption controls and IRQ locking:
 *   Even if the pre-emption is enabled, tasks will be forced to pend if
 *   the IRQ lock is also set UNLESS the CPU starting the task is the
 *   holder of the IRQ lock.
 *
 * Input Parameters:
 *   rtcb - Points to the blocked TCB that is ready-to-run
 *
 * Returned Value:
 *   true  - IRQs are locked by a different CPU.
 *   false - IRQs are unlocked OR if they are locked BUT this CPU
 *           is the holder of the lock.
 *
 *   Warning: This values are volatile at only valid at the instance that
 *   the CPU set was queried.
 *
 ****************************************************************************/

#ifdef CONFIG_SMP
bool irq_cpu_locked(int cpu)
{
  cpu_set_t irqset;

  /* g_cpu_irqset is not valid in early phases of initialization */

  if (g_os_initstate < OSINIT_OSREADY)
    {
      /* We are still single threaded.  In either state of g_cpu_irqlock,
       * the correct return value should always be false.
       */

      return false;
    }

#if defined(CONFIG_ARCH_HAVE_FETCHADD) && !defined(CONFIG_ARCH_GLOBAL_IRQDISABLE)
  /* If the global lockcount has been incremented then simply return true */

  if (g_global_lockcount > 0)
    {
      return true;
    }
#endif

  /* Test if g_cpu_irqlock is locked.  We don't really need to use check
   * g_cpu_irqlock to do this, we can use the g_cpu_set.
   *
   * Sample the g_cpu_irqset once.  That is an atomic operation.  All
   * subsequent operations will operate on the sampled cpu set.
   */

  irqset = (cpu_set_t)g_cpu_irqset;
  if (irqset != 0)
    {
      /* Some CPU holds the lock.  So g_cpu_irqlock should be locked.
       * Return false if the 'cpu' is the holder of the lock; return
       * true if g_cpu_irqlock is locked, but this CPU is not the
       * holder of the lock.
       */

      return ((irqset & (1 << cpu)) == 0);
    }

  /* No CPU holds the lock */

  else
    {
      /* In this case g_cpu_irqlock should be unlocked.  However, if
       * the lock was established in the interrupt handler AND there are
       * no bits set in g_cpu_irqset, that probably means only that
       * critical section was established from an interrupt handler.
       * Return false in either case.
       */

      return false;
    }
}
#endif

#endif /* CONFIG_IRQCOUNT */