这是indexloc提供的服务,不要输入任何密码
Skip to content

Fails on incompressible data #196

@nskyav

Description

@nskyav

Some compressors (brieflz, density, fastlz, tornado, ucl_nrv2b, ucl_nrv2d, ucl_nrv2e, zlib, zlib-ng) fail on incompressible data (like dd if=/dev/random of=somefile bs=1024 count=1024) after 7b98759.
Partial revert of 7b98759 like below helps:

diff --git a/bench/lzbench.cpp b/bench/lzbench.cpp
index 74a4329..9006ed5 100644
--- a/bench/lzbench.cpp
+++ b/bench/lzbench.cpp
@@ -288,11 +288,15 @@ inline int64_t lzbench_compress(lzbench_params_t *params, std::vector<size_t>& c
.
         clen = compress((char*)inbuf, part, (char*)outbuf, outpart, codec_options);
.
-        if (clen <= 0)
+        if (clen <= 0 || clen == part)
         {
-            LZBENCH_PRINT(9, "ERROR: part=%lu clen=%ld in=%lu out=%lu\n", (uint64_t)part, clen, (uint64_t)(inbuf-start), (uint64_t)sum);
-            LZBENCH_PRINT(0, "ERROR: compression error in=%zu out=%ld/%zu (in_bytes=%lu out_bytes=%ld)\n", part, clen, outpart, (uint64_t)(inbuf+part-start), (int64_t)sum+clen);
-            return 0;
+            if (part > outsize) {
+                LZBENCH_PRINT(9, "ERROR: part=%lu clen=%ld in=%lu out=%lu\n", (uint64_t)part, clen, (uint64_t)(inbuf-start), (uint64_t)sum);
+                LZBENCH_PRINT(0, "ERROR: compression error in=%zu out=%ld/%zu (in_bytes=%lu out_bytes=%ld)\n", part, clen, outpart, (uint64_t)(inbuf+part-start), (int64_t)sum+clen);
+                return 0;
+            }
+            memcpy(outbuf, inbuf, part);
+            clen = part;
         }
.
         inbuf += part;
@@ -324,7 +328,13 @@ inline int64_t lzbench_decompress(lzbench_params_t *params, std::vector<size_t>&
         }
#endif
.
-        dlen = decompress((char*)inbuf, part, (char*)outbuf, chunk_sizes[i], codec_options);
+        if (part == chunk_sizes[i]) // uncompressed
+        {
+            memcpy(outbuf, inbuf, part);
+            dlen = part;
+        } else {
+            dlen = decompress((char*)inbuf, part, (char*)outbuf, chunk_sizes[i], codec_options);
+        }
.
         if (dlen <= 0) {
             LZBENCH_PRINT(9, "DEC part=%lu dlen=%ld out=%lu\n", (uint64_t)part, dlen, (uint64_t)(outbuf - outstart));

There are similar fails with crush, lzo2a, bzip3, tamp, yalz77, but not sure if it's related to 7b98759 and incompressible data.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions