arm64: SME2 GEMM and level-3 kernels for Apple M4 (VORTEXM4)

CONTRIBUTORS.md

@@ -290,5 +290,9 @@ hheei <hheei@users.noreply.github.com>
 * Hugo Meiland <hugo@meiland.nl>
   * [2026-08-09] Add Cortex-A72 DGEMM 6x8 microkernel and blocking
 
+* Michael Tesch <https://github.com/tesch1>
+  * [2026-09-29] SME2 SGEMM/DGEMM kernel for Apple M4
+  * [2026-09-29] SME2 level-3 kernels (GEMM, TRMM and TRSM kernels and panel copies) for Apple M4
+
 * Arthur031221 <levi74108520963@gmail.com>
   * [2026-10-02] Fix complex scaling in strided batched GEMM.

README.md

@@ -187,7 +187,7 @@ Please read `GotoBLAS_01Readme.txt` for older CPU models already supported by th
 - **Neoverse V3**: preliminary support
 - **Neoverse V3AE**: preliminary support
 - **Apple Vortex**: preliminary support based on ThunderX2/3
-- **Apple VortexM4**: preliminary support based on ThunderX2/3, SME kernels for SGEMM,SSYMM,STRMM,SSYRK,SSYR2K
+- **Apple VortexM4**: preliminary support based on ThunderX2/3, SME kernels for SGEMM,SSYMM,STRMM,SSYRK,SSYR2K; on SME2 cores with a 512-bit streaming vector length (tested on M4) SME2 kernels for SGEMM and DGEMM and for the S/D level-3 driver (SYMM, SYRK, SYR2K, TRMM, TRSM and the LAPACK routines built on them)
 - **A64FX**:  preliminary support, optimized Level-3 BLAS
 - **ARMV8SVE**: any ARMV8 cpu with SVE extensions 
 - **ARMV9SME**: any ARMV9 cpu with SVE and SME extensions 

cmake/prebuild.cmake

@@ -1273,10 +1273,17 @@ endif ()
       "#define L2_ASSOCIATIVE\t8\n"
       "#define DTB_DEFAULT_ENTRIES\t64\n"
       "#define DTB_SIZE\t4096\n")
-    set(SGEMM_UNROLL_M 16)
-    set(SGEMM_UNROLL_N 4)
-    set(DGEMM_UNROLL_M 8)
-    set(DGEMM_UNROLL_N 4)
+    if ("${TCORE}" STREQUAL "VORTEXM4" AND NOT "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+      set(SGEMM_UNROLL_M 64)
+      set(SGEMM_UNROLL_N 16)
+      set(DGEMM_UNROLL_M 64)
+      set(DGEMM_UNROLL_N 8)
+    else ()
+      set(SGEMM_UNROLL_M 16)
+      set(SGEMM_UNROLL_N 4)
+      set(DGEMM_UNROLL_M 8)
+      set(DGEMM_UNROLL_N 4)
+    endif ()
     set(CGEMM_UNROLL_M 8)
     set(CGEMM_UNROLL_N 4)
     set(ZGEMM_UNROLL_M 4)

common_thread.h

@@ -148,6 +148,33 @@ extern int blas_omp_number_max;
 extern int blas_omp_threads_local;
 extern int blas_is_num_threads_set_explicitly;
 
+#if defined(ARCH_ARM64) && defined(OS_DARWIN) && defined(__clang__) && (defined(VORTEXM4) || defined(DYNAMIC_ARCH)) && \
+    !defined(COMPLEX) && !defined(BFLOAT16) && !defined(HFLOAT16)
+#include <sys/sysctl.h>
+#ifdef DYNAMIC_ARCH
+extern char *gotoblas_corename(void);
+#endif
+/* Apple M4: real level-3 BLAS and LAPACK run on one SME unit per P-cluster; measured on M4 Pro only, not on M5+. */
+static __inline int sme_level3_threads(int nthreads) {
+  static int units = 0;
+#ifdef DYNAMIC_ARCH
+  if (strcmp(gotoblas_corename(), "vortexm4") != 0) return nthreads;
+#endif
+  if (units == 0) {
+    int cpus = 0, per = 0, u = 1;
+    size_t len = sizeof(int);
+    if (sysctlbyname("hw.perflevel0.physicalcpu", &cpus, &len, NULL, 0) == 0 &&
+        (len = sizeof(int), sysctlbyname("hw.perflevel0.cpusperl2", &per, &len, NULL, 0) == 0) && per > 0 &&
+        cpus / per > 1)
+      u = cpus / per;
+    units = u;
+  }
+  return nthreads < units ? nthreads : units;
+}
+#else
+#define sme_level3_threads(nthreads) (nthreads)
+#endif
+
 static __inline int num_cpu_avail(int level) {
 
 #ifdef USE_OPENMP
@@ -157,7 +184,7 @@ static __inline int num_cpu_avail(int level) {
      in case the user hasn't made an explicit choice). */
   if (blas_is_num_threads_set_explicitly) {
 	  if (omp_in_parallel()) return 1;
-    return blas_cpu_number;
+    return level >= 3 ? sme_level3_threads(blas_cpu_number) : blas_cpu_number;
   }
 
 int openmp_nthreads;
@@ -184,7 +211,7 @@ int openmp_nthreads;
   }
 #endif
 
-  return blas_cpu_number;
+  return level >= 3 ? sme_level3_threads(blas_cpu_number) : blas_cpu_number;
 
 }
 

cpuid_arm64.c

@@ -408,13 +408,15 @@ int detect(void)
 	sysctlbyname("hw.cpufamily",&value64,&length64,NULL,0);
 	if (value64 ==131287967|| value64 == 458787763 ) return CPU_VORTEX; //A12/M1
 	if (value64 == 3660830781) return CPU_VORTEX; //A15/M2
-        if (value64 == 2271604202) return CPU_VORTEX; //A16/M3
-        if (value64 == 1867590060) return CPU_VORTEXM4; //M4
-	if (value64 == 492472296) return CPU_VORTEXM4; //M5
-	if (support_sme1())
-	    return CPU_VORTEXM4;
-	else
-	    return CPU_VORTEX;	
+  if (value64 == 2271604202) return CPU_VORTEX; //A16/M3
+  if (value64 == 1867590060) return CPU_VORTEXM4; //M4
+  if (value64 == 399882554) return CPU_VORTEXM4; //M4 Pro/Max
+  if (value64 == 492472296) return CPU_VORTEXM4; //M5
+	{ /* later Apple cores with SME: same kernels until they get their own entry */
+	int sme = 0;
+	size_t len = sizeof(sme);
+	if (sysctlbyname("hw.optional.arm.FEAT_SME", &sme, &len, NULL, 0) == 0 && sme) return CPU_VORTEXM4;
+  }
 #else
 #ifdef OS_WINDOWS
 	HKEY reghandle;

kernel/CMakeLists.txt

@@ -1144,27 +1144,38 @@ endif ()
     endforeach ()
 
     if ((BUILD_DOUBLE OR BUILD_COMPLEX) AND NOT BUILD_SINGLE)
+      if (NOT DEFINED TRSMCOPYLN_M)
+        set(STRSMUNCOPY_M "generic/trsm_uncopy_${SGEMM_UNROLL_M}.c")
+        set(STRSMLNCOPY_M "generic/trsm_lncopy_${SGEMM_UNROLL_M}.c")
+        set(STRSMUTCOPY_M "generic/trsm_utcopy_${SGEMM_UNROLL_M}.c")
+        set(STRSMLTCOPY_M "generic/trsm_ltcopy_${SGEMM_UNROLL_M}.c")
+      else ()
+        set(STRSMUNCOPY_M "${KERNELDIR}/${TRSMCOPYUN_M}")
+        set(STRSMLNCOPY_M "${KERNELDIR}/${TRSMCOPYLN_M}")
+        set(STRSMUTCOPY_M "${KERNELDIR}/${TRSMCOPYUT_M}")
+        set(STRSMLTCOPY_M "${KERNELDIR}/${TRSMCOPYLT_M}")
+      endif ()
 	    GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_LN}" "UPPER;LN;TRSMKERNEL" "trsm_kernel_LN" false "" "" false "SINGLE")
 	    GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_LT}" "LT;TRSMKERNEL" "trsm_kernel_LT" false "" "" false "SINGLE")
 	    GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_RN}" "UPPER;RN;TRSMKERNEL" "trsm_kernel_RN" false "" "" false "SINGLE")
 	    GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_RT}" "RT;TRSMKERNEL" "trsm_kernel_RT" false "" "" false "SINGLE")
-      GenerateNamedObjects("generic/trsm_uncopy_${SGEMM_UNROLL_M}.c" "UNIT" "trsm_iunucopy" false "" "" false "SINGLE")
-      GenerateNamedObjects("generic/trsm_uncopy_${SGEMM_UNROLL_M}.c" "" "trsm_iunncopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMUNCOPY_M} "UNIT" "trsm_iunucopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMUNCOPY_M} "" "trsm_iunncopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_uncopy_${SGEMM_UNROLL_N}.c" "OUTER;UNIT" "trsm_ounucopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_uncopy_${SGEMM_UNROLL_N}.c" "OUTER" "trsm_ounncopy" false "" "" false "SINGLE")
 
-      GenerateNamedObjects("generic/trsm_lncopy_${SGEMM_UNROLL_M}.c" "LOWER;UNIT" "trsm_ilnucopy" false "" "" false "SINGLE")
-      GenerateNamedObjects("generic/trsm_lncopy_${SGEMM_UNROLL_M}.c" "LOWER" "trsm_ilnncopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMLNCOPY_M} "LOWER;UNIT" "trsm_ilnucopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMLNCOPY_M} "LOWER" "trsm_ilnncopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_lncopy_${SGEMM_UNROLL_N}.c" "OUTER;LOWER;UNIT" "trsm_olnucopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_lncopy_${SGEMM_UNROLL_N}.c" "OUTER;LOWER" "trsm_olnncopy" false "" "" false "SINGLE")
 
-      GenerateNamedObjects("generic/trsm_utcopy_${SGEMM_UNROLL_M}.c" "UNIT" "trsm_iutucopy" false "" "" false "SINGLE")
-      GenerateNamedObjects("generic/trsm_utcopy_${SGEMM_UNROLL_M}.c" "" "trsm_iutncopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMUTCOPY_M} "UNIT" "trsm_iutucopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMUTCOPY_M} "" "trsm_iutncopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_utcopy_${SGEMM_UNROLL_N}.c" "OUTER;UNIT" "trsm_outucopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_utcopy_${SGEMM_UNROLL_N}.c" "OUTER" "trsm_outncopy" false "" "" false "SINGLE")
 
-      GenerateNamedObjects("generic/trsm_ltcopy_${SGEMM_UNROLL_M}.c" "LOWER;UNIT" "trsm_iltucopy" false "" "" false "SINGLE")
-      GenerateNamedObjects("generic/trsm_ltcopy_${SGEMM_UNROLL_M}.c" "LOWER" "trsm_iltncopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMLTCOPY_M} "LOWER;UNIT" "trsm_iltucopy" false "" "" false "SINGLE")
+      GenerateNamedObjects(${STRSMLTCOPY_M} "LOWER" "trsm_iltncopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_ltcopy_${SGEMM_UNROLL_N}.c" "OUTER;LOWER;UNIT" "trsm_oltucopy" false "" "" false "SINGLE")
       GenerateNamedObjects("generic/trsm_ltcopy_${SGEMM_UNROLL_N}.c" "OUTER;LOWER" "trsm_oltncopy" false "" "" false "SINGLE")
       if (SMALL_MATRIX_OPT)

kernel/arm64/KERNEL.VORTEXM4

@@ -1 +1,54 @@
 include $(KERNELDIR)/KERNEL.NEOVERSEN1
+
+# SME2 level-3 kernels (GEMM_UNROLL_M 64, GEMM_UNROLL_N one streaming vector); GCC builds keep the NEON kernels
+ifneq ($(C_COMPILER), GCC)
+SGEMMKERNEL    = gemm_kernel_sme2.c
+STRMMKERNEL    = gemm_kernel_sme2.c
+SGEMMINCOPY    = gemm_ncopy_sme2.c
+SGEMMITCOPY    = gemm_tcopy_sme2.c
+SGEMMONCOPY    = ../generic/gemm_ncopy_$(SGEMM_UNROLL_N).c
+SGEMMOTCOPY    = ../generic/gemm_tcopy_$(SGEMM_UNROLL_N).c
+SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX)
+SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX)
+SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX)
+SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
+DGEMMKERNEL    = gemm_kernel_sme2.c
+DTRMMKERNEL    = gemm_kernel_sme2.c
+DGEMMINCOPY    = gemm_ncopy_sme2.c
+DGEMMITCOPY    = gemm_tcopy_sme2.c
+DGEMMONCOPY    = ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c
+DGEMMOTCOPY    = ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c
+DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX)
+DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX)
+DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX)
+DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
+STRSMKERNEL_LN = trsm_kernel_sme2.c
+STRSMKERNEL_LT = trsm_kernel_sme2.c
+STRSMKERNEL_RN = trsm_kernel_sme2.c
+STRSMKERNEL_RT = trsm_kernel_sme2.c
+DTRSMKERNEL_LN = trsm_kernel_sme2.c
+DTRSMKERNEL_LT = trsm_kernel_sme2.c
+DTRSMKERNEL_RN = trsm_kernel_sme2.c
+DTRSMKERNEL_RT = trsm_kernel_sme2.c
+
+TRSMCOPYUN_M   = trsm_ncopy_sme2.c
+TRSMCOPYLN_M   = trsm_ncopy_sme2.c
+TRSMCOPYUT_M   = trsm_tcopy_sme2.c
+TRSMCOPYLT_M   = trsm_tcopy_sme2.c
+
+STRMMUNCOPY_M  = trmm_ncopy_sme2.c
+STRMMLNCOPY_M  = trmm_ncopy_sme2.c
+STRMMUTCOPY_M  = trmm_tcopy_sme2.c
+STRMMLTCOPY_M  = trmm_tcopy_sme2.c
+DTRMMUNCOPY_M  = trmm_ncopy_sme2.c
+DTRMMLNCOPY_M  = trmm_ncopy_sme2.c
+DTRMMUTCOPY_M  = trmm_tcopy_sme2.c
+DTRMMLTCOPY_M  = trmm_tcopy_sme2.c
+
+SSYMMUCOPY_M   = symm_copy_sme2.c
+SSYMMLCOPY_M   = symm_copy_sme2.c
+DSYMMUCOPY_M   = symm_copy_sme2.c
+DSYMMLCOPY_M   = symm_copy_sme2.c
+endif

kernel/arm64/gemm_kernel_sme2.c

@@ -0,0 +1,161 @@
+/***************************************************************************
+Copyright (c) 2026, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/* SME2 GEMM kernel of the level-3 driver, the TRMM kernel with TRMMKERNEL; plain loops without SME2. */
+
+#include "common.h"
+#include "sme2_gemm_detect.h"
+#include "sme2_gemm_packed.h"
+
+/* Depth range of the block at (i0, j0): the TRMM kernel skips the zero part of the triangle as trmmkernel_4x4.c. */
+static inline void s2p_krange(BLASLONG i0, BLASLONG j0, int wm, int wn, BLASLONG k, BLASLONG offset, BLASLONG *ks,
+                              BLASLONG *kl) S2P_SC {
+#ifdef TRMMKERNEL
+#ifdef LEFT
+  const BLASLONG off = offset + i0;
+#else
+  const BLASLONG off = j0 - offset;
+#endif
+#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+  BLASLONG s = off, e = k;
+#elif defined(LEFT)
+  BLASLONG s = 0, e = off + wm;
+#else
+  BLASLONG s = 0, e = off + wn;
+#endif
+  if (s < 0) s = 0;
+  if (s > k) s = k;
+  if (e > k) e = k;
+  if (e < s) e = s;
+  *ks = s;
+  *kl = e - s;
+#else
+  *ks = 0;
+  *kl = k;
+#endif
+}
+
+#ifdef HAVE_SME2_GEMM
+#pragma clang attribute push(__attribute__((target("sme2"))), apply_to = function)
+
+typedef void (*s2p_bfn)(BLASLONG, const FLOAT *, int, const FLOAT *, int, FLOAT *, BLASLONG, FLOAT, int) S2_S S2_ZA;
+
+/* Block of wm rows and wn columns of C; cm 0: C += A B, 1: C -= A B, 2: C += alpha A B, 3: C = alpha A B. */
+#define S2P_DEF(TN, BF, SUB)                                                                                     \
+  S2_NOINL void s2p_blk_##TN##_##BF##_##SUB(BLASLONG kb, const FLOAT *a, int wm, const FLOAT *b, int wn, FLOAT *C, \
+                                            BLASLONG ldc, FLOAT alpha, int cm) S2_S S2_ZA {                      \
+    if (cm < 2) s2_c_load(1, TN, C, ldc, wn, wm, S2_LOAD, (FLOAT)1);                                             \
+    else svzero_za();                                                                                            \
+    s2p_mac(TN, BF, SUB, kb, a, wm, b, wn);                                                                      \
+    if (cm < 2) s2_c_store(1, TN, C, ldc, wn, wm);                                                               \
+    else s2_c_store_alpha(TN, C, ldc, wn, wm, alpha, cm == 2);                                                   \
+  }
+#define S2P_DEF4(TN) S2P_DEF(TN, 0, 0) S2P_DEF(TN, 0, 1) S2P_DEF(TN, 1, 0) S2P_DEF(TN, 1, 1)
+#define S2P_ROW(TN) {{s2p_blk_##TN##_0_0, s2p_blk_##TN##_0_1}, {s2p_blk_##TN##_1_0, s2p_blk_##TN##_1_1}}
+S2P_DEF4(1)
+S2P_DEF4(2)
+S2P_DEF4(3)
+S2P_DEF4(4)
+#ifdef DOUBLE
+S2P_DEF4(5)
+S2P_DEF4(6)
+S2P_DEF4(7)
+S2P_DEF4(8)
+static const s2p_bfn s2p_blk[S2_NT][2][2] = {S2P_ROW(1), S2P_ROW(2), S2P_ROW(3), S2P_ROW(4),
+                                             S2P_ROW(5), S2P_ROW(6), S2P_ROW(7), S2P_ROW(8)};
+#else
+static const s2p_bfn s2p_blk[S2_NT][2][2] = {S2P_ROW(1), S2P_ROW(2), S2P_ROW(3), S2P_ROW(4)};
+#endif
+
+__arm_new("za") __arm_locally_streaming static void s2p_drive(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha,
+                                                              const FLOAT *sa, const FLOAT *sb, FLOAT *C,
+                                                              BLASLONG ldc, BLASLONG offset) {
+#ifdef TRMMKERNEL
+  const int cm = 3;
+#else
+  const int cm = alpha == (FLOAT)1 ? 0 : (alpha == (FLOAT)-1 ? 1 : 2);
+#endif
+  for (BLASLONG j0 = 0; j0 < n;) {
+    const int wn = s2p_wn(n, j0);
+    for (BLASLONG i0 = 0; i0 < m; i0 += S2_NR) {
+      const int wm = m - i0 < S2_NR ? (int)(m - i0) : S2_NR;
+      BLASLONG ks, kl;
+      s2p_krange(i0, j0, wm, wn, k, offset, &ks, &kl);
+      s2p_blk[(wm + S2_VL - 1) / S2_VL - 1][wn == S2_VL][cm == 1](kl, sa + i0 * k + ks * wm, wm,
+                                                                  sb + j0 * k + ks * wn, wn, C + i0 + j0 * ldc,
+                                                                  ldc, alpha, cm);
+    }
+    j0 += wn;
+  }
+}
+
+#pragma clang attribute pop
+#endif
+
+static void s2p_loops(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, const FLOAT *sa, const FLOAT *sb, FLOAT *C,
+                      BLASLONG ldc, BLASLONG offset) {
+  for (BLASLONG j0 = 0; j0 < n;) {
+    const int wn = s2p_wn(n, j0);
+    for (BLASLONG i0 = 0; i0 < m; i0 += S2P_UM) {
+      const int wm = m - i0 < S2P_UM ? (int)(m - i0) : S2P_UM;
+      BLASLONG ks, kl;
+      s2p_krange(i0, j0, wm, wn, k, offset, &ks, &kl);
+      const FLOAT *a = sa + i0 * k + ks * wm, *b = sb + j0 * k + ks * wn;
+      for (int j = 0; j < wn; ++j)
+        for (int i = 0; i < wm; ++i) {
+          FLOAT s = 0;
+          for (BLASLONG l = 0; l < kl; ++l) s += a[l * wm + i] * b[l * wn + j];
+#ifdef TRMMKERNEL
+          C[i0 + i + (j0 + j) * ldc] = alpha * s;
+#else
+          C[i0 + i + (j0 + j) * ldc] += alpha * s;
+#endif
+        }
+    }
+    j0 += wn;
+  }
+}
+
+int CNAME(BLASLONG bm, BLASLONG bn, BLASLONG bk, FLOAT alpha, FLOAT *ba, FLOAT *bb, FLOAT *C, BLASLONG ldc
+#ifdef TRMMKERNEL
+          , BLASLONG offset
+#endif
+) {
+#ifndef TRMMKERNEL
+  const BLASLONG offset = 0;
+  if (bk <= 0) return 0;
+#endif
+  if (bm <= 0 || bn <= 0) return 0;
+#ifdef HAVE_SME2_GEMM
+  if (s2_usable()) {
+    s2p_drive(bm, bn, bk, alpha, ba, bb, C, ldc, offset);
+    return 0;
+  }
+#endif
+  s2p_loops(bm, bn, bk, alpha, ba, bb, C, ldc, offset);
+  return 0;
+}