diff --git a/rtl/ibex_core.sv b/rtl/ibex_core.sv
index 3e1919628..b8f693ca4 100644
--- a/rtl/ibex_core.sv
+++ b/rtl/ibex_core.sv
@@ -618,6 +618,7 @@ module ibex_core import ibex_pkg::*; #(
     // CSR ID/EX
     .csr_access_o         (csr_access),
     .csr_op_o             (csr_op),
+    .csr_addr_o           (csr_addr),
     .csr_op_en_o          (csr_op_en),
     .csr_save_if_o        (csr_save_if),  // control signal to save PC
     .csr_save_id_o        (csr_save_id),  // control signal to save PC
@@ -1046,7 +1047,6 @@ module ibex_core import ibex_pkg::*; #(
   /////////////////////////////////////////
 
   assign csr_wdata  = alu_operand_a_ex;
-  assign csr_addr   = csr_num_e'(csr_access ? alu_operand_b_ex[11:0] : 12'b0);
 
   ibex_cs_registers #(
     .DbgTriggerEn     (DbgTriggerEn),
diff --git a/rtl/ibex_decoder.sv b/rtl/ibex_decoder.sv
index 4b0195930..6e925f565 100644
--- a/rtl/ibex_decoder.sv
+++ b/rtl/ibex_decoder.sv
@@ -83,6 +83,7 @@ module ibex_decoder #(
   // CSRs
   output logic                 csr_access_o,          // access to CSR
   output ibex_pkg::csr_op_e    csr_op_o,              // operation to perform on CSR
+  output ibex_pkg::csr_num_e   csr_addr_o,            // CSR address
 
   // LSU
   output logic                 data_req_o,            // start transaction to data memory
@@ -138,6 +139,8 @@ module ibex_decoder #(
   assign imm_u_type_o = { instr[31:12], 12'b0 };
   assign imm_j_type_o = { {12{instr[31]}}, instr[19:12], instr[20], instr[30:21], 1'b0 };
 
+  assign csr_addr_o = csr_num_e'(instr[31:20]);
+
   // immediate for CSR manipulation (zero extended)
   assign zimm_rs1_type_o = { 27'b0, instr_rs1 }; // rs1
 
@@ -1168,9 +1171,10 @@ module ibex_decoder #(
           alu_op_b_mux_sel_o = OP_B_IMM;
         end else begin
           // instruction to read/modify CSR
-          alu_op_b_mux_sel_o = OP_B_IMM;
           imm_a_mux_sel_o    = IMM_A_Z;
-          imm_b_mux_sel_o    = IMM_B_I;  // CSR address is encoded in I imm
+
+          // No need for operand/immediate B mux selection. The CSR address is fed out as csr_addr_o
+          // as the CSR address always comes from the same field in the instruction.
 
           if (instr_alu[14]) begin
             // rs1 field is used as immediate
diff --git a/rtl/ibex_id_stage.sv b/rtl/ibex_id_stage.sv
index 647e648b1..43f7762f1 100644
--- a/rtl/ibex_id_stage.sv
+++ b/rtl/ibex_id_stage.sv
@@ -95,6 +95,7 @@ module ibex_id_stage #(
   // CSR
   output logic                      csr_access_o,
   output ibex_pkg::csr_op_e         csr_op_o,
+  output ibex_pkg::csr_num_e        csr_addr_o,
   output logic                      csr_op_en_o,
   output logic                      csr_save_if_o,
   output logic                      csr_save_id_o,
@@ -290,6 +291,7 @@ module ibex_id_stage #(
   logic        data_req_allowed;
 
   // CSR control
+  logic        no_flush_csr_addr;
   logic        csr_pipe_flush;
 
   logic [31:0] alu_operand_a;
@@ -497,6 +499,7 @@ module ibex_id_stage #(
     // CSRs
     .csr_access_o(csr_access_o),
     .csr_op_o    (csr_op_o),
+    .csr_addr_o  (csr_addr_o),
 
     // LSU
     .data_req_o           (lsu_req_dec),
@@ -509,36 +512,20 @@ module ibex_id_stage #(
     .branch_in_dec_o(branch_in_dec)
   );
 
-  /////////////////////////////////
-  // CSR-related pipeline flushes //
-  /////////////////////////////////
-  always_comb begin : csr_pipeline_flushes
-    csr_pipe_flush = 1'b0;
-
-    // A pipeline flush is needed to let the controller react after modifying certain CSRs:
-    // - When enabling interrupts, pending IRQs become visible to the controller only during
-    //   the next cycle. If during that cycle the core disables interrupts again, it does not
-    //   see any pending IRQs and consequently does not start to handle interrupts.
-    // - When modifying any PMP CSR, PMP check of the next instruction might get invalidated.
-    //   Hence, a pipeline flush is needed to instantiate another PMP check with the updated CSRs.
-    // - When modifying debug CSRs.
-    if (csr_op_en_o == 1'b1 && (csr_op_o == CSR_OP_WRITE || csr_op_o == CSR_OP_SET)) begin
-      if (csr_num_e'(instr_rdata_i[31:20]) == CSR_MSTATUS ||
-          csr_num_e'(instr_rdata_i[31:20]) == CSR_MIE     ||
-          csr_num_e'(instr_rdata_i[31:20]) == CSR_MSECCFG ||
-          // To catch all PMPCFG/PMPADDR registers, get the shared top most 7 bits.
-          instr_rdata_i[31:25] == 7'h1D) begin
-        csr_pipe_flush = 1'b1;
-      end
-    end else if (csr_op_en_o == 1'b1 && csr_op_o != CSR_OP_READ) begin
-      if (csr_num_e'(instr_rdata_i[31:20]) == CSR_DCSR      ||
-          csr_num_e'(instr_rdata_i[31:20]) == CSR_DPC       ||
-          csr_num_e'(instr_rdata_i[31:20]) == CSR_DSCRATCH0 ||
-          csr_num_e'(instr_rdata_i[31:20]) == CSR_DSCRATCH1) begin
-        csr_pipe_flush = 1'b1;
-      end
-    end
-  end
+  // Flush pipe on most CSR modification. Some CSR modifications alter how instructions execute
+  // (e.g. the PMP CSRs) so this ensures all instructions always see the latest architectural state
+  // when entering the fetch stage. This causes some needless flushes but performance impact is
+  // limited. We have a single fetch stage to flush not many stages of a deep pipeline and CSR
+  // instructions are in general rare and not part of performance critical parts of the code.
+  //
+  // No flush is triggered for a small number of specific CSRs. These are ones that have been
+  // specifically identified to be a) likely to be modifed in exception handlers and b) safe to
+  // alter without a flush.
+  assign no_flush_csr_addr = csr_addr_o inside {CSR_MSCRATCH, CSR_MEPC};
+
+  assign csr_pipe_flush = (csr_op_en_o == 1)                                         &&
+                          (csr_op_o inside {CSR_OP_WRITE, CSR_OP_SET, CSR_OP_CLEAR}) &&
+                          !no_flush_csr_addr;
 
   ////////////////
   // Controller //