path: root/bsps/riscv/riscv/irq/irq.c

/* SPDX-License-Identifier: BSD-2-Clause */

/**
 * @file
 *
 * @ingroup riscv_interrupt
 *
 * @brief Interrupt support.
 */

/*
 * Copyright (C) 2018, 2022 embedded brains GmbH & Co. KG
 *
 * Copyright (c) 2015 University of York.
 * Hesham Almatary <hesham@alumni.york.ac.uk>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <bsp/irq.h>
#include <bsp/fatal.h>
#include <bsp/fdt.h>
#include <bsp/irq-generic.h>
#include <bsp/riscv.h>

#include <rtems/score/percpu.h>
#include <rtems/score/riscv-utility.h>
#include <rtems/score/scheduler.h>
#include <rtems/score/smpimpl.h>

#include <libfdt.h>

static volatile RISCV_PLIC_regs *riscv_plic;

volatile RISCV_CLINT_regs *riscv_clint;

#ifdef RTEMS_SMP
/*
 * The lovely PLIC has an interrupt enable bit per hart for each interrupt
 * source.  This makes the interrupt enable/disable a bit difficult.  We have
 * to store the interrupt distribution in software.  To keep it simple, we
 * support only a one-to-one and one-to-all interrupt to processor
 * distribution.  For a one-to-one distribution, the array member must point to
 * the enable register block of the corresponding.  For a one-to-all
 * distribution, the array member must be NULL.  The array index is the
 * external interrupt index minus one (external interrupt index zero is a
 * special value, see PLIC documentation).
 */
static volatile uint32_t *
riscv_plic_irq_to_cpu[RISCV_MAXIMUM_EXTERNAL_INTERRUPTS];
#endif

RTEMS_INTERRUPT_LOCK_DEFINE(static, riscv_plic_lock, "PLIC")

void _RISCV_Interrupt_dispatch(uintptr_t mcause, Per_CPU_Control *cpu_self)
{
  /*
   * Get rid of the most significant bit which indicates if the exception was
   * caused by an interrupt or not.
   */
  mcause <<= 1;

  if (mcause == (RISCV_INTERRUPT_TIMER_MACHINE << 1)) {
    bsp_interrupt_handler_dispatch_unchecked(RISCV_INTERRUPT_VECTOR_TIMER);
  } else if (mcause == (RISCV_INTERRUPT_EXTERNAL_MACHINE << 1)) {
    volatile RISCV_PLIC_hart_regs *plic_hart_regs;
    uint32_t interrupt_index;

    plic_hart_regs = cpu_self->cpu_per_cpu.plic_hart_regs;

    while ((interrupt_index = plic_hart_regs->claim_complete) != 0) {
      bsp_interrupt_handler_dispatch(
        RISCV_INTERRUPT_VECTOR_EXTERNAL(interrupt_index)
      );

      plic_hart_regs->claim_complete = interrupt_index;

      /*
       * FIXME: It is not clear which fence is necessary here or if a fence is
       * necessary at all.  The goal is that the complete signal is somehow
       * recognized by the PLIC before the next claim is issued.
       */
      __asm__ volatile ("fence o, i" : : : "memory");
    }
  } else if (mcause == (RISCV_INTERRUPT_SOFTWARE_MACHINE << 1)) {
    /*
     * Clear the software interrupt on this processor.  Synchronization of
     * inter-processor interrupts is done via Per_CPU_Control::message in
     * _SMP_Inter_processor_interrupt_handler().
     */
    *cpu_self->cpu_per_cpu.clint_msip = 0;

#ifdef RTEMS_SMP
    _SMP_Inter_processor_interrupt_handler(cpu_self);
    bsp_interrupt_handler_dispatch_unlikely(RISCV_INTERRUPT_VECTOR_SOFTWARE);
#else
    bsp_interrupt_handler_dispatch_unchecked(RISCV_INTERRUPT_VECTOR_SOFTWARE);
#endif
  } else {
    bsp_fatal(RISCV_FATAL_UNEXPECTED_INTERRUPT_EXCEPTION);
  }
}

static void riscv_clint_per_cpu_init(
  volatile RISCV_CLINT_regs *clint,
  Per_CPU_Control *cpu,
  uint32_t index
)
{
  cpu->cpu_per_cpu.clint_msip = &clint->msip[index];
  cpu->cpu_per_cpu.clint_mtimecmp = &clint->mtimecmp[index];
}

static void riscv_plic_per_cpu_init(
  volatile RISCV_PLIC_regs *plic,
  uint32_t enable_register_count,
  Per_CPU_Control *cpu,
  uint32_t index
)
{
  volatile uint32_t *enable;
  uint32_t i;

  plic->harts[index].priority_threshold = 0;

  enable = &plic->enable[index][0];
  cpu->cpu_per_cpu.plic_m_ie = enable;
  cpu->cpu_per_cpu.plic_hart_regs = &plic->harts[index];

  for (i = 0; i < enable_register_count; ++i) {
    enable[i] = 0;
  }
}

static void riscv_plic_cpu_0_init(
  volatile RISCV_PLIC_regs *plic,
  uint32_t interrupt_last
)
{
#ifdef RTEMS_SMP
  Per_CPU_Control *cpu;
#endif
  uint32_t i;

#ifdef RTEMS_SMP
  cpu = _Per_CPU_Get_by_index(0);
#endif

  for (i = 1; i <= interrupt_last; ++i) {
    plic->priority[i] = 1;
#ifdef RTEMS_SMP
    riscv_plic_irq_to_cpu[i - 1] = cpu->cpu_per_cpu.plic_m_ie;
#endif
  }

  /*
   * External M-mode interrupts on secondary processors are enabled in
   * bsp_start_on_secondary_processor().
   */
  set_csr(mie, MIP_MEIP);
}

static void riscv_clint_init(const void *fdt)
{
  volatile RISCV_CLINT_regs *clint;
  int node;
  const uint32_t *val;
  int len;
  int i;

  node = fdt_node_offset_by_compatible(fdt, -1, "riscv,clint0");

  clint = riscv_fdt_get_address(fdt, node);
  if (clint == NULL) {
    bsp_fatal(RISCV_FATAL_NO_CLINT_REG_IN_DEVICE_TREE);
  }

  riscv_clint = clint;

  val = fdt_getprop(fdt, node, "interrupts-extended", &len);

  for (i = 0; i < len; i += 16) {
    uint32_t hart_index;
    uint32_t cpu_index;

    hart_index = riscv_get_hart_index_by_phandle(fdt32_to_cpu(val[i / 4]));

#ifdef RTEMS_SMP
    cpu_index = _RISCV_Map_hardid_to_cpu_index(hart_index);
    if (cpu_index >= rtems_configuration_get_maximum_processors()) {
      continue;
    }

    if ( _Scheduler_Initial_assignments[ cpu_index ].scheduler == NULL ) {
      /* Skip not configured processor */
      continue;
    }
#else
    if (hart_index != RISCV_BOOT_HARTID) {
      continue;
    }

    cpu_index = 0;
#endif

    riscv_clint_per_cpu_init(
      clint,
      _Per_CPU_Get_by_index(cpu_index),
      (uint32_t) (i / 16)
    );

#ifndef RTEMS_SMP
    break;
#endif
  }
}

static void riscv_plic_init(const void *fdt)
{
  volatile RISCV_PLIC_regs *plic;
  int node;
  int i;
  const uint32_t *val;
  int len;
  uint32_t interrupt_index;
  uint32_t ndev;

  node = fdt_node_offset_by_compatible(fdt, -1, "riscv,plic0");

  plic = riscv_fdt_get_address(fdt, node);

  if (plic == NULL) {
#ifdef RISCV_ENABLE_HTIF_SUPPORT
    node = fdt_node_offset_by_compatible(fdt, -1, "ucb,htif0");

    /* Spike platform has HTIF and does not have a PLIC */
    if (node >= 0) {
      return;
    } else {
      bsp_fatal(RISCV_FATAL_NO_PLIC_REG_IN_DEVICE_TREE);
    }
#else
    bsp_fatal(RISCV_FATAL_NO_PLIC_REG_IN_DEVICE_TREE);
#endif
  }

  riscv_plic = plic;

  val = fdt_getprop(fdt, node, "riscv,ndev", &len);
  if (val == NULL || len != 4) {
    bsp_fatal(RISCV_FATAL_INVALID_PLIC_NDEV_IN_DEVICE_TREE);
  }

  ndev = fdt32_to_cpu(val[0]);
  if (ndev > RISCV_MAXIMUM_EXTERNAL_INTERRUPTS) {
    bsp_fatal(RISCV_FATAL_TOO_LARGE_PLIC_NDEV_IN_DEVICE_TREE);
  }

  val = fdt_getprop(fdt, node, "interrupts-extended", &len);

  for (i = 0; i < len; i += 8) {
    uint32_t hart_index;
    uint32_t enable_register_count;
    uint32_t cpu_index;

    /*
     * Each interrupt enable register contains exactly 32 enable bits.
     * Calculate the enable register count based on the number of interrupts
     * supported by the PLIC.  Take the reserved interrupt ID zero into
     * account.
     */
    enable_register_count = RTEMS_ALIGN_UP(ndev + 1, 32) / 32;

    hart_index = riscv_get_hart_index_by_phandle(fdt32_to_cpu(val[i / 4]));

#ifdef RTEMS_SMP
    cpu_index = _RISCV_Map_hardid_to_cpu_index(hart_index);
    if (cpu_index >= rtems_configuration_get_maximum_processors()) {
      continue;
    }

    if ( _Scheduler_Initial_assignments[ cpu_index ].scheduler == NULL ) {
      /* Skip not configured processor */
      continue;
    }
#else
    if (hart_index != RISCV_BOOT_HARTID) {
      continue;
    }

    cpu_index = 0;
#endif

    interrupt_index = fdt32_to_cpu(val[i / 4 + 1]);
    if (interrupt_index != RISCV_INTERRUPT_EXTERNAL_MACHINE) {
      continue;
    }

    riscv_plic_per_cpu_init(
      plic,
      enable_register_count,
      _Per_CPU_Get_by_index(cpu_index),
      (uint32_t) (i / 8)
    );

#ifndef RTEMS_SMP
    break;
#endif
  }

  riscv_plic_cpu_0_init(plic, ndev);
}

void bsp_interrupt_facility_initialize(void)
{
  const void *fdt;

  fdt = bsp_fdt_get();
  riscv_clint_init(fdt);
  riscv_plic_init(fdt);
}

bool bsp_interrupt_is_valid_vector(rtems_vector_number vector)
{
  /*
   * The PLIC interrupt ID of zero is reserved.  For example, this ID is used
   * to indicate that no interrupt was claimed.
   */
  if (vector == RISCV_INTERRUPT_VECTOR_EXTERNAL(0)) {
    return false;
  }

  return vector < (rtems_vector_number) BSP_INTERRUPT_VECTOR_COUNT;
}

rtems_status_code bsp_interrupt_get_attributes(
  rtems_vector_number         vector,
  rtems_interrupt_attributes *attributes
)
{
  attributes->is_maskable = true;
  attributes->can_enable = true;
  attributes->maybe_enable = true;
  attributes->can_disable = true;
  attributes->maybe_disable = true;
  attributes->can_raise = (vector == RISCV_INTERRUPT_VECTOR_SOFTWARE);
  attributes->can_raise_on = attributes->can_raise;
  attributes->cleared_by_acknowledge = true;
  attributes->can_get_affinity = RISCV_INTERRUPT_VECTOR_IS_EXTERNAL(vector);
  attributes->can_set_affinity = attributes->can_get_affinity;

  if (vector == RISCV_INTERRUPT_VECTOR_SOFTWARE) {
    attributes->trigger_signal = RTEMS_INTERRUPT_NO_SIGNAL;
  }

  return RTEMS_SUCCESSFUL;
}

rtems_status_code bsp_interrupt_is_pending(
  rtems_vector_number vector,
  bool               *pending
)
{
  bsp_interrupt_assert(bsp_interrupt_is_valid_vector(vector));
  bsp_interrupt_assert(pending != NULL);

  if (RISCV_INTERRUPT_VECTOR_IS_EXTERNAL(vector)) {
    uint32_t interrupt_index;
    uint32_t group;
    uint32_t bit;

    interrupt_index = RISCV_INTERRUPT_VECTOR_EXTERNAL_TO_INDEX(vector);
    group = interrupt_index / 32;
    bit = UINT32_C(1) << (interrupt_index % 32);
    *pending = ((riscv_plic->pending[group] & bit) != 0);
    return RTEMS_SUCCESSFUL;
  }

  if (vector == RISCV_INTERRUPT_VECTOR_TIMER) {
    *pending = (read_csr(mip) & MIP_MTIP) != 0;
    return RTEMS_SUCCESSFUL;
  }

  _Assert(vector == RISCV_INTERRUPT_VECTOR_SOFTWARE);
  *pending = (read_csr(mip) & MIP_MSIP) != 0;
  return RTEMS_SUCCESSFUL;
}

static inline rtems_status_code riscv_raise_on(
  rtems_vector_number vector,
  uint32_t            cpu_index
)
{
  Per_CPU_Control *cpu;

  bsp_interrupt_assert(bsp_interrupt_is_valid_vector(vector));

  if (vector != RISCV_INTERRUPT_VECTOR_SOFTWARE) {
    return RTEMS_UNSATISFIED;
  }

  cpu = _Per_CPU_Get_by_index(cpu_index);
  *cpu->cpu_per_cpu.clint_msip = 0x1;
  return RTEMS_SUCCESSFUL;
}

rtems_status_code bsp_interrupt_raise(rtems_vector_number vector)
{
  return riscv_raise_on(vector, rtems_scheduler_get_processor());
}

#if defined(RTEMS_SMP)
rtems_status_code bsp_interrupt_raise_on(
  rtems_vector_number vector,
  uint32_t            cpu_index
)
{
  return riscv_raise_on(vector, cpu_index);
}
#endif

rtems_status_code bsp_interrupt_clear(rtems_vector_number vector)
{
  bsp_interrupt_assert(bsp_interrupt_is_valid_vector(vector));
  return RTEMS_UNSATISFIED;
}

rtems_status_code bsp_interrupt_vector_is_enabled(
  rtems_vector_number vector,
  bool               *enabled
)
{
  bsp_interrupt_assert(bsp_interrupt_is_valid_vector(vector));
  bsp_interrupt_assert(enabled != NULL);

  if (RISCV_INTERRUPT_VECTOR_IS_EXTERNAL(vector)) {
    uint32_t interrupt_index;
    uint32_t group;
    uint32_t bit;
    Per_CPU_Control *cpu;
#ifdef RTEMS_SMP
    uint32_t cpu_max;
    uint32_t cpu_index;
#endif

    interrupt_index = RISCV_INTERRUPT_VECTOR_EXTERNAL_TO_INDEX(vector);
    group = interrupt_index / 32;
    bit = UINT32_C(1) << (interrupt_index % 32);

#ifdef RTEMS_SMP
    cpu_max = _SMP_Get_processor_maximum();

    for (cpu_index = 0; cpu_index < cpu_max; ++cpu_index) {
      volatile uint32_t *enable;

      cpu = _Per_CPU_Get_by_index(cpu_index);
      enable = cpu->cpu_per_cpu.plic_m_ie;

      if (enable != NULL && (enable[group] & bit) != 0) {
        *enabled = true;
        return RTEMS_SUCCESSFUL;
      }
    }

    *enabled = false;
#else
    cpu = _Per_CPU_Get_by_index(0);
    *enabled = (cpu->cpu_per_cpu.plic_m_ie[group] & bit) != 0;
#endif

    return RTEMS_SUCCESSFUL;
  }

  if (vector == RISCV_INTERRUPT_VECTOR_TIMER) {
    *enabled = (read_csr(mie) & MIP_MTIP) != 0;
    return RTEMS_SUCCESSFUL;
  }

  _Assert(vector == RISCV_INTERRUPT_VECTOR_SOFTWARE);
  *enabled = (read_csr(mie) & MIP_MSIP) != 0;
  return RTEMS_SUCCESSFUL;
}

rtems_status_code bsp_interrupt_vector_enable(rtems_vector_number vector)
{
  bsp_interrupt_assert(bsp_interrupt_is_valid_vector(vector));

  if (RISCV_INTERRUPT_VECTOR_IS_EXTERNAL(vector)) {
    uint32_t interrupt_index;
    uint32_t group;
    uint32_t bit;
    rtems_interrupt_lock_context lock_context;
    Per_CPU_Control *cpu;
#ifdef RTEMS_SMP
    volatile uint32_t *enable;
#endif

    interrupt_index = RISCV_INTERRUPT_VECTOR_EXTERNAL_TO_INDEX(vector);
    group = interrupt_index / 32;
    bit = UINT32_C(1) << (interrupt_index % 32);
#ifdef RTEMS_SMP
    enable = riscv_plic_irq_to_cpu[interrupt_index - 1];
#endif

    rtems_interrupt_lock_acquire(&riscv_plic_lock, &lock_context);

#ifdef RTEMS_SMP
    if (enable != NULL) {
      enable[group] |= bit;
    } else {
      uint32_t cpu_max;
      uint32_t cpu_index;

      cpu_max = _SMP_Get_processor_maximum();

      for (cpu_index = 0; cpu_index < cpu_max; ++cpu_index) {
        cpu = _Per_CPU_Get_by_index(cpu_index);
        enable = cpu->cpu_per_cpu.plic_m_ie;

        if (enable != NULL) {
          enable[group] |= bit;
        }
      }
    }
#else
    cpu = _Per_CPU_Get_by_index(0);
    cpu->cpu_per_cpu.plic_m_ie[group] |= bit;
#endif

    rtems_interrupt_lock_release(&riscv_plic_lock, &lock_context);
    return RTEMS_SUCCESSFUL;
  }

  if (vector == RISCV_INTERRUPT_VECTOR_TIMER) {
    set_csr(mie, MIP_MTIP);
    return RTEMS_SUCCESSFUL;
  }

  _Assert(vector == RISCV_INTERRUPT_VECTOR_SOFTWARE);
  set_csr(mie, MIP_MSIP);
  return RTEMS_SUCCESSFUL;
}

rtems_status_code bsp_interrupt_vector_disable(rtems_vector_number vector)
{
  bsp_interrupt_assert(bsp_interrupt_is_valid_vector(vector));

  if (RISCV_INTERRUPT_VECTOR_IS_EXTERNAL(vector)) {
    uint32_t interrupt_index;
    uint32_t group;
    uint32_t bit;
    rtems_interrupt_lock_context lock_context;
    Per_CPU_Control *cpu;
#ifdef RTEMS_SMP
    volatile uint32_t *enable;
#endif

    interrupt_index = RISCV_INTERRUPT_VECTOR_EXTERNAL_TO_INDEX(vector);
    group = interrupt_index / 32;
    bit = UINT32_C(1) << (interrupt_index % 32);
#ifdef RTEMS_SMP
    enable = riscv_plic_irq_to_cpu[interrupt_index - 1];
#endif

    rtems_interrupt_lock_acquire(&riscv_plic_lock, &lock_context);

#ifdef RTEMS_SMP
    if (enable != NULL) {
      enable[group] &= ~bit;
    } else {
      uint32_t cpu_max;
      uint32_t cpu_index;

      cpu_max = _SMP_Get_processor_maximum();

      for (cpu_index = 0; cpu_index < cpu_max; ++cpu_index) {
        cpu = _Per_CPU_Get_by_index(cpu_index);
        enable = cpu->cpu_per_cpu.plic_m_ie;

        if (enable != NULL) {
          enable[group] &= ~bit;
        }
      }
    }
#else
    cpu = _Per_CPU_Get_by_index(0);
    cpu->cpu_per_cpu.plic_m_ie[group] &= ~bit;
#endif

    rtems_interrupt_lock_release(&riscv_plic_lock, &lock_context);
    return RTEMS_SUCCESSFUL;
  }

  if (vector == RISCV_INTERRUPT_VECTOR_TIMER) {
    clear_csr(mie, MIP_MTIP);
    return RTEMS_SUCCESSFUL;
  }

  _Assert(vector == RISCV_INTERRUPT_VECTOR_SOFTWARE);
  clear_csr(mie, MIP_MSIP);
  return RTEMS_SUCCESSFUL;
}

#ifdef RTEMS_SMP
rtems_status_code bsp_interrupt_set_affinity(
  rtems_vector_number vector,
  const Processor_mask *affinity
)
{
  if (RISCV_INTERRUPT_VECTOR_IS_EXTERNAL(vector)) {
    uint32_t interrupt_index;
    Processor_mask mask;

    interrupt_index = RISCV_INTERRUPT_VECTOR_EXTERNAL_TO_INDEX(vector);

    _Processor_mask_And(&mask, affinity, _SMP_Get_online_processors());

    if (_Processor_mask_Is_equal(&mask, _SMP_Get_online_processors())) {
      riscv_plic_irq_to_cpu[interrupt_index - 1] = NULL;
      return RTEMS_SUCCESSFUL;
    }

    if (_Processor_mask_Count(&mask) == 1) {
      uint32_t cpu_index;
      Per_CPU_Control *cpu;

      cpu_index = _Processor_mask_Find_last_set(&mask) - 1;
      cpu = _Per_CPU_Get_by_index(cpu_index);
      riscv_plic_irq_to_cpu[interrupt_index - 1] = cpu->cpu_per_cpu.plic_m_ie;
      return RTEMS_SUCCESSFUL;
    }

    return RTEMS_INVALID_NUMBER;
  }

  return RTEMS_UNSATISFIED;
}

rtems_status_code bsp_interrupt_get_affinity(
  rtems_vector_number vector,
  Processor_mask *affinity
)
{
  if (RISCV_INTERRUPT_VECTOR_IS_EXTERNAL(vector)) {
    uint32_t interrupt_index;
    volatile uint32_t *enable;

    interrupt_index = RISCV_INTERRUPT_VECTOR_EXTERNAL_TO_INDEX(vector);
    enable = riscv_plic_irq_to_cpu[interrupt_index - 1];

    if (enable != NULL) {
      uint32_t cpu_max;
      uint32_t cpu_index;

      cpu_max = _SMP_Get_processor_maximum();

      for (cpu_index = 0; cpu_index < cpu_max; ++cpu_index) {
        Per_CPU_Control *cpu;

        cpu = _Per_CPU_Get_by_index(cpu_index);

        if (enable == cpu->cpu_per_cpu.plic_m_ie) {
          _Processor_mask_Set(affinity, cpu_index);
          break;
        }
      }
    } else {
      _Processor_mask_Assign(affinity, _SMP_Get_online_processors());
    }

    return RTEMS_SUCCESSFUL;
  }

  return RTEMS_UNSATISFIED;
}
#endif