use crc32c checksum to validate .ji cache files (#21240)

* use crc32c checksum to validate .ji cache files (for #21154)

base/loading.jl

@@ -680,7 +680,15 @@ function compilecache(name::String)
             info("Precompiling module $name.")
         end
     end
-    if !success(create_expr_cache(path, cachefile, concrete_deps))
+    if success(create_expr_cache(path, cachefile, concrete_deps))
+        # append checksum to the end of the .ji file:
+        open(cachefile, "a+") do f
+            data = Mmap.mmap(f, Vector{UInt8}, filesize(f), 0)
+            checksum = crc32c(data)
+            finalize(data)
+            write(f, hton(checksum))
+        end
+    else
         error("Failed to precompile $name to $cachefile.")
     end
     return cachefile
@@ -799,6 +807,18 @@ function stale_cachefile(modpath::String, cachefile::String)
                 return true
             end
         end
+
+        # finally, verify that the cache file has a valid checksum
+        data = Mmap.mmap(io, Vector{UInt8}, filesize(io), 0)
+        # checksum = UInt32 read in bigendian format from the last 4 bytes:
+        checksum = UInt32(data[end]) + UInt32(data[end-1])<<8 + UInt32(data[end-2])<<16 + UInt32(data[end-3])<<24
+        crc = crc32c(@view(data[1:end-4]))
+        finalize(data)
+        if checksum != crc
+            DEBUG_LOADING[] && info("JL_DEBUG_LOADING: Rejecting cache file $cachefile because it has an invalid checksum.")
+            return true
+        end
+
         return false # fresh cachefile
     finally
         close(io)

base/util.jl

@@ -765,16 +765,22 @@ if is_windows()
 
 end
 
+# compute sizeof correctly for strings, arrays, and subarrays of bytes
+_sizeof(a) = sizeof(a)
+_sizeof(a::FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N) = length(a)
+
 """
     crc32c(data, crc::UInt32=0x00000000)
+
 Compute the CRC-32c checksum of the given `data`, which can be
-an `Array{UInt8}` or a `String`.  Optionally, you can pass
-a starting `crc` integer to be mixed in with the checksum.
+an `Array{UInt8}`, a contiguous subarray thereof, or a `String`.  Optionally, you can pass
+a starting `crc` integer to be mixed in with the checksum.  The `crc` parameter
+can be used to compute a checksum on data divided into chunks: performing
+`crc32c(data2, crc32c(data1))` is equivalent to the checksum of `[data1; data2]`.
 (Technically, a little-endian checksum is computed.)
 """
-function crc32c end
-crc32c(a::Union{Array{UInt8},String}, crc::UInt32=0x00000000) =
-    ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, a, sizeof(a))
+crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N,String}, crc::UInt32=0x00000000) =
+    ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, a, _sizeof(a))
 
 """
     @kwdef typedef

test/compile.jl

@@ -310,6 +310,12 @@ try
     @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji"))
     @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji"))
 
+    # test checksum
+    open(joinpath(dir2, "FooBar1.ji"), "a") do f
+        write(f, 0x076cac96) # append 4 random bytes
+    end
+    @test Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji"))
+
     # test behavior of precompile modules that throw errors
     write(FooBar_file,
           """

test/misc.jl

@@ -568,6 +568,13 @@ for force_software_crc in (1,0)
     for (n,crc) in [(0,0x00000000),(1,0xa016d052),(2,0x03f89f52),(3,0xf130f21e),(4,0x29308cf4),(5,0x53518fab),(6,0x4f4dfbab),(7,0xbd3a64dc),(8,0x46891f81),(9,0x5a14b9f9),(10,0xb219db69),(11,0xd232a91f),(12,0x51a15563),(13,0x9f92de41),(14,0x4d8ae017),(15,0xc8b74611),(16,0xa0de6714),(17,0x672c992a),(18,0xe8206eb6),(19,0xc52fd285),(20,0x327b0397),(21,0x318263dd),(22,0x08485ccd),(23,0xea44d29e),(24,0xf6c0cb13),(25,0x3969bba2),(26,0x6a8810ec),(27,0x75b3d0df),(28,0x82d535b1),(29,0xbdf7fc12),(30,0x1f836b7d),(31,0xd29f33af),(32,0x8e4acb3e),(33,0x1cbee2d1),(34,0xb25f7132),(35,0xb0fa484c),(36,0xb9d262b4),(37,0x3207fe27),(38,0xa024d7ac),(39,0x49a2e7c5),(40,0x0e2c157f),(41,0x25f7427f),(42,0x368c6adc),(43,0x75efd4a5),(44,0xa84c5c31),(45,0x0fc817b2),(46,0x8d99a881),(47,0x5cc3c078),(48,0x9983d5e2),(49,0x9267c2db),(50,0xc96d4745),(51,0x058d8df3),(52,0x453f9cf3),(53,0xb714ade1),(54,0x55d3c2bc),(55,0x495710d0),(56,0x3bddf494),(57,0x4f2577d0),(58,0xdae0f604),(59,0x3c57c632),(60,0xfe39bbb0),(61,0x6f5d1d41),(62,0x7d996665),(63,0x68c738dc),(64,0x8dfea7ae)]
         @test Base.crc32c(UInt8[1:n;]) == crc
     end
+    # test that crc parameter is equivalent to checksum of concatenated data,
+    # and test crc of subarrays:
+    a = UInt8[1:255;]
+    crc_256 = Base.crc32c(UInt8[1:255;])
+    @views for n = 1:255
+        @test Base.crc32c(a[n+1:end], Base.crc32c(a[1:n])) == crc_256
+    end
 end
 
 let