feat(ci): rework Dockerfile to use cargo chef (#90)

* Use cargo chef in Dockerfile to prevent frequent full rebuilds
* Fix .env loading issue for Docker-based launch ("path not found")

.dockerignore

@@ -6,4 +6,6 @@ node-modules
 ledger
 .anchor
 docker/geyser-outputs
-docker/solana-outputs
+docker/solana-outputs
+snapshot/
+.env

Dockerfile

@@ -1,11 +1,26 @@
-FROM rust:1.83.0
-
-RUN apt-get update && apt install -y git curl protobuf-compiler
+# This Dockerfile describes the build process for the snapshot
+# ETL tool.
 
+FROM rust:1.83.0-bullseye AS chef
+RUN cargo install cargo-chef
 WORKDIR /app
 
-COPY . ./
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+FROM chef AS builder 
+COPY --from=planner /app/recipe.json recipe.json
+RUN apt update && apt install -y git curl protobuf-compiler
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release --recipe-path recipe.json
+# Build application
+COPY . .
+RUN cargo build --release --bin solana-snapshot-etl -F standalone
 
-RUN cargo build --release -p plerkle
+FROM debian:bullseye-slim AS runtime
+RUN apt update && apt install openssl
+WORKDIR /app
+COPY --from=builder /app/target/release/solana-snapshot-etl .
 
-ENTRYPOINT ["cargo", "r", "--features=standalone", "--package=plerkle_snapshot", "--bin=solana-snapshot-etl"]
+ENTRYPOINT ["./solana-snapshot-etl"]

Makefile

@@ -13,4 +13,4 @@ build:
 	@docker build -f Dockerfile . -t ${IMAGE_NAME}
 
 stream:
-	for f in $(shell ls ${SNAPSHOTDIR}); do echo $$(realpath $${f}) && docker run --env-file .env -p 3000:3000 --rm -it --mount type=bind,source=$$(realpath $${f}),target=$$(realpath $${f}),readonly --mount type=bind,source=$$(pwd)/etl-config.json,target=/app/etl-config.json,readonly ${IMAGE_NAME} $$(realpath $${f}) --geyser=./etl-config.json && date; done
+	for f in $(shell ls ${SNAPSHOTDIR}); do echo $$(realpath $${f}) && docker run --env-file .env -p 3000:3000 --rm -it --mount type=bind,source=$$(realpath $${f}),target=$$(realpath $${f}),readonly --mount type=bind,source=$$(pwd)/etl-config.json,target=/app/etl-config.json,readonly ${IMAGE_NAME} $$(realpath $${f}) --geyser=/app/etl-config.json && date; done

plerkle_snapshot/src/bin/solana-snapshot-etl/main.rs

@@ -12,7 +12,7 @@ use serde::Deserialize;
 use std::fs::File;
 use std::io::{IoSliceMut, Read};
 use std::path::Path;
-use tracing::info;
+use tracing::{info, warn};
 
 mod geyser;
 mod mpl_metadata;
@@ -28,13 +28,15 @@ struct Args {
 
 #[tokio::main(flavor = "multi_thread")]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    dotenvy::dotenv()?;
-
     let env_filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string());
     tracing_subscriber::fmt()
         .with_env_filter(env_filter)
         .event_format(tracing_subscriber::fmt::format::json())
         .init();
+    if let Err(e) = dotenvy::dotenv() {
+        warn!(error = %e, "Error initializing .env, make sure all required env is available...");
+    }
+
     let args = Args::parse();
 
     let mut loader = SupportedLoader::new(&args.source, Box::new(LoadProgressTracking {}))?;