Initial Snowflake Arrow prototype

.cargo/config

@@ -0,0 +1,15 @@
+[target.x86_64-unknown-linux-gnu]
+linker = "clang"
+rustflags = ["-C", "link-arg=-fuse-ld=mold", "-C", "target-cpu=native"]
+#rustflags = ["-C", "link-arg=-fuse-ld=mold"]
+
+[target.'cfg(target_os = "macos")']
+rustflags = [
+    "-C", "link-arg=-fuse-ld=/usr/local/bin/zld",
+    "-C", "link-arg=-undefined",
+    "-C", "link-arg=dynamic_lookup"
+]
+
+[profile.release]
+lto = true
+codegen-units = 1

.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

.gitignore

@@ -0,0 +1,30 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+snowflake_arrow-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/
+target
+.idea
+/priv/native/*so
+*.iml

README.md

@@ -1 +1,43 @@
-Please check the Pull Request!
+# SnowflakeArrow
+
+Snowflake specific arrow implementation, which only implements the [Data types Snowflake supports](https://docs.snowflake.com/en/sql-reference/data-types.html).
+
+This has been designed to work with [req_snowflake](https://github.com/joshuataylor/req_snowflake), which when added to a project with `req_snowflake` will
+allow Snowflake to send back Arrow results which this library will parse.
+
+This library uses [Rustler](https://github.com/rusterlium/rustler) to create a Rust binding to [Arrow2](https://github.com/jorgecarleitao/arrow2) library.
+
+Arrow2 has been chosen as the performance is incredible (from my basic rust knowledge I could get some very good results), and is being used more and more.
+
+Nothing against arrow-rs, I just went with Arrow2.
+
+## Installation
+
+```elixir
+def deps do
+  [
+    {:snowflake_arrow, github: "joshuataylor/snowflake_arrow"}
+  ]
+end
+```
+
+## Features
+
+- Optional setting to cast to Elixir types from within Rust, which will allow you to not have to cast them yourself (if not set, will cast to strings, integers, floats)
+- 
+
+## Benchmarks
+
+Benchmarks are hard to compare against the JSON implementation of Snowflake, as we also have to cast and do other things to get the same results.
+
+I have benchmarks against the files Snowflake sends back (they send back files ranging from a few kb to 20mb), and the parsing results are pretty incredible.
+Once we get the library cleaned up, public benchmarks will be made available, comparing the full deserialization of the JSON using Elixir to the Rust implementation with
+casting to Elixir Types.
+
+## Help please
+
+My Rust knowledge is basic, so I just went with what felt right. I am not a Rustacean (yet! 🦀), so please feel free to suggest improvements.
+
+Please see the pull request for my comments around the quick Rust implementation, I wrote this in a day after studying and playing with Arrow & Arrow2.
+
+It's also single threaded, as I'm unsure how much improvement could be made with multi-threading.

lib/benchmark.exs

@@ -0,0 +1,14 @@
+arrow_data_small = File.read!("benchmark/small_arrow")
+arrow_data_large = File.read!("benchmark/large_arrow")
+jason_data_large = File.read!("benchmark/large_json.json")
+jason_data_small = File.read!("benchmark/small_json.json")
+
+Benchee.run(
+  %{
+    "arrow_small (368kb)" => fn -> SnowflakeArrow.Native.convert_arrow_stream(arrow_data_small, true) end,
+    "arrow_large (9.4mb)" => fn -> SnowflakeArrow.Native.convert_arrow_stream(arrow_data_large, true) end,
+    "jason_decode (8.8mb)" => fn -> Jason.decode!("[#{jason_data_large}]") end,
+    "jason_decode (468Kb)" => fn -> Jason.decode!("[#{jason_data_small}]") end,
+  },
+  time: 5
+)

lib/snowflake_arrow.ex

@@ -0,0 +1,18 @@
+defmodule SnowflakeArrow do
+  @moduledoc """
+  Documentation for `SnowflakeArrow`.
+  """
+
+  @doc """
+  Hello world.
+
+  ## Examples
+
+      iex> SnowflakeArrow.hello()
+      :world
+
+  """
+  def hello do
+    :world
+  end
+end

lib/snowflake_arrow/snowflake_arrow_native.ex

@@ -0,0 +1,8 @@
+defmodule SnowflakeArrow.Native do
+  use Rustler, otp_app: :snowflake_arrow, crate: :snowflake_arrow, mode: :release
+
+
+  def convert_arrow_stream(_options, _cast), do: error()
+  #  def convert_arrow_stream_no_cast(_options), do: error()
+  defp error, do: :erlang.nif_error(:nif_not_loaded)
+end

lib/snowflake_arrow/td.ex

@@ -0,0 +1,3 @@
+defmodule TestDate do
+  defstruct [:year, :month, :day]
+end

mix.exs

@@ -0,0 +1,27 @@
+defmodule SnowflakeArrow.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :snowflake_arrow,
+      version: "0.1.0",
+      elixir: "~> 1.13",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications.
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  defp deps do
+    [
+      {:rustler, "~> 0.25.0"},
+      {:benchee, "~> 1.1"}
+    ]
+  end
+end

mix.lock

@@ -0,0 +1,8 @@
+%{
+  "benchee": {:hex, :benchee, "1.1.0", "f3a43817209a92a1fade36ef36b86e1052627fd8934a8b937ac9ab3a76c43062", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}], "hexpm", "7da57d545003165a012b587077f6ba90b89210fd88074ce3c60ce239eb5e6d93"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
+  "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"},
+  "rustler": {:hex, :rustler, "0.25.0", "32526b51af7e58a740f61941bf923486ce6415a91c3934cc16c281aa201a2240", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "6b43a11a37fe79c6234d88c4102ab5dfede7a6a764dc5c7b539956cfa02f3cf4"},
+  "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
+  "toml": {:hex, :toml, "0.6.2", "38f445df384a17e5d382befe30e3489112a48d3ba4c459e543f748c2f25dd4d1", [:mix], [], "hexpm", "d013e45126d74c0c26a38d31f5e8e9b83ea19fc752470feb9a86071ca5a672fa"},
+}

native/snowflake_arrow/.cargo/config

@@ -0,0 +1,15 @@
+[target.x86_64-unknown-linux-gnu]
+linker = "clang"
+#rustflags = ["-C", "link-arg=-fuse-ld=mold", "-C", "target-cpu=native"]
+rustflags = ["-C", "link-arg=-fuse-ld=mold"]
+
+[target.'cfg(target_os = "macos")']
+rustflags = [
+    "-C", "link-arg=-fuse-ld=/usr/local/bin/zld",
+    "-C", "link-arg=-undefined",
+    "-C", "link-arg=dynamic_lookup"
+]
+
+[profile.release]
+lto = true
+codegen-units = 1

native/snowflake_arrow/.gitignore

@@ -0,0 +1 @@
+/target