RFC: DataSets.jl intergration

Data.toml

@@ -0,0 +1,11 @@
+data_config_version=1
+
+[[datasets]]
+name="Pi"
+uuid="76e82d27-3914-4472-abb6-38d476023211"
+
+    [datasets.storage]
+    driver="DataDeps"
+    sha256="1f08958df70c30bbf3f7205ad5f9b2b9430e3378b4e4e6300063bd9fbd83d6e3"
+    localdir="~/.julia/datadeps/Pi"
+    remote_path=["https://www.angio.net/pi/digits/10000.txt"]

Project.toml

@@ -5,6 +5,7 @@ version = "0.7.7"
 
 [deps]
 BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
+DataSets = "c9661210-8a83-48f0-b833-72e62abce419"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"

src/DataDeps.jl

@@ -42,4 +42,12 @@ include("preupload.jl")
 include("fetch_helpers.jl")
 include("post_fetch_helpers.jl")
 
+# Extension
+using DataSets
+include("datasets_storage_driver.jl")
+
+function __init__()
+    DataSets.add_storage_driver("DataDeps"=>DataDep_storage_driver)
+end
+
 end # module

src/datasets_storage_driver.jl

@@ -0,0 +1,21 @@
+# storage driver for DataSets.jl
+
+function DataDep_storage_driver(user_func, storage_config, dataset)
+    conf = dataset.conf
+    dp = DataDep(
+        conf["name"],
+        "",
+        storage_config["remote_path"],
+        get(storage_config, "sha256", "")
+    )
+    localdir = expanduser(storage_config["localdir"])
+    if !isdir(localdir)
+        try
+            DataDeps.download(dp, localdir; i_accept_the_terms_of_use=true)
+        catch
+            @error "failed to download dataset"
+            rm(localdir; recursive=true)
+        end
+    end
+    user_func(BlobTree(DataSets.FileSystemRoot(localdir), path"."))
+end