drop in rvv variable-length-memset

Simon Hosie · Simon Hosie · commit 7487ef919b83 · 2023-11-08T12:43:15.000-08:00
diff --git a/arch/riscv/chunkset_rvv.c b/arch/riscv/chunkset_rvv.c
@@ -29,6 +29,22 @@ do {                                                                    \
     } while (len > 0);                                                  \
 } while (0)
 
+static inline void* memcpy_rvv(void* dst, void const* src, size_t len) {
+    // TODO: m1, m2, m4, or m8 might provide the best balance of predictable
+    // branches against unnecessary operations.  Tune this.
+    do {
+        size_t vl =  __riscv_vsetvl_e8m2(len);
+        vuint8m2_t chunk = __riscv_vle8_v_u8m2(src, vl);
+        __riscv_vse8_v_u8m2(dst, chunk, vl);
+        dst = (uint8_t*)dst + vl;
+        src = (uint8_t*)src + vl;
+        len -= vl;
+    } while (len > 0);
+    return dst;
+}
+
+#define VARLEN_MEMCPY memcpy_rvv
+
 /* We don't have a 32-byte datatype for RISC-V arch. */
 typedef struct chunk_s {
     uint64_t data[4];
@@ -93,14 +109,14 @@ static inline uint8_t* CHUNKCOPY(uint8_t *out, uint8_t const *from, unsigned len
     len -= align;
     ptrdiff_t dist = out - from;
     if (dist >= len) {
-        memcpy(out, from, len);
+        memcpy_rvv(out, from, len);
         out += len;
         from += len;
         return out;
     }
     if (dist >= sizeof(chunk_t)) {
         dist = (dist / sizeof(chunk_t)) * sizeof(chunk_t);
-        memcpy(out, from, dist);
+        memcpy_rvv(out, from, dist);
         out += dist;
         from += dist;
         len -= dist;
diff --git a/chunkset_tpl.h b/chunkset_tpl.h
@@ -9,6 +9,10 @@
 extern uint8_t* chunkmemset_ssse3(uint8_t *out, unsigned dist, unsigned len);
 #endif
 
+#if !defined VARLEN_MEMCPY
+#define VARLEN_MEMCPY memcpy
+#endif
+
 /* Returns the chunk size */
 Z_INTERNAL uint32_t CHUNKSIZE(void) {
     return sizeof(chunk_t);
@@ -79,7 +83,7 @@ static inline chunk_t GET_CHUNK_MAG(uint8_t *buf, uint32_t *chunk_rem, uint32_t
         uint8_t *cur_chunk = (uint8_t *)&chunk_load;
         while (bytes_remaining) {
             cpy_dist = MIN(dist, bytes_remaining);
-            memcpy(cur_chunk, buf, cpy_dist);
+            VARLEN_MEMCPY(cur_chunk, buf, cpy_dist);
             bytes_remaining -= cpy_dist;
             cur_chunk += cpy_dist;
             /* This allows us to bypass an expensive integer division since we're effectively
@@ -161,7 +165,7 @@ Z_INTERNAL uint8_t* CHUNKMEMSET(uint8_t *out, unsigned dist, unsigned len) {
     }
 
     if (len) {
-        memcpy(out, &chunk_load, len);
+        VARLEN_MEMCPY(out, &chunk_load, len);
         out += len;
     }
 
diff --git a/inflate_p.h b/inflate_p.h
@@ -39,6 +39,10 @@
 #  define INFLATE_GET_DICTIONARY_HOOK(strm, dict, dict_len) do {} while (0)
 #endif
 
+#if !defined VARLEN_MEMCPY
+#define VARLEN_MEMCPY memcpy
+#endif
+
 /*
  *   Macros shared by inflate() and inflateBack()
  */
@@ -157,7 +161,7 @@ static inline uint8_t* chunkcopy_safe(uint8_t *out, uint8_t *from, uint64_t len,
 
     /* For all cases without overlap, memcpy is ideal */
     if (!(olap_src || olap_dst)) {
-        memcpy(out, from, (size_t)len);
+        VARLEN_MEMCPY(out, from, (size_t)len);
         return out + len;
     }
 
@@ -173,7 +177,7 @@ static inline uint8_t* chunkcopy_safe(uint8_t *out, uint8_t *from, uint64_t len,
      * behind or lookahead distance. */
     uint64_t non_olap_size = llabs(from - out); // llabs vs labs for compatibility with windows
 
-    memcpy(out, from, (size_t)non_olap_size);
+    VARLEN_MEMCPY(out, from, (size_t)non_olap_size);
     out += non_olap_size;
     from += non_olap_size;
     len -= non_olap_size;
