Skip to content

Commit

Permalink
feat: support preheat with self-signed certs (#3541)
Browse files Browse the repository at this point in the history
Signed-off-by: Gaius <[email protected]>
  • Loading branch information
gaius-qi authored Sep 26, 2024
1 parent 1afe79e commit ea850f7
Show file tree
Hide file tree
Showing 14 changed files with 529 additions and 819 deletions.
48 changes: 24 additions & 24 deletions deploy/docker-compose/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ services:
- MARIADB_DATABASE=manager
- MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=yes
healthcheck:
test: ["CMD-SHELL", "mysqladmin status"]
test: ["CMD-SHELL", "mysqladmin ping -h localhost"]
interval: 1s
timeout: 2s
retries: 30
Expand All @@ -31,13 +31,13 @@ services:

manager:
image: dragonflyoss/manager:latest
container_name: dragonfly-manager
container_name: manager
depends_on:
- redis
- mysql
restart: always
volumes:
- ./log/manager:/var/log/dragonfly/manager
- ./log/manager:/var/log/dragonfly
- ./config/manager.yaml:/etc/dragonfly/manager.yaml:ro
healthcheck:
test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:65003 || exit 1"]
Expand All @@ -48,60 +48,60 @@ services:
- 65003:65003
- 8080:8080

dfdaemon:
image: dragonflyoss/dfdaemon:latest
client:
image: dragonflyoss/client:latest
depends_on:
- manager
- scheduler
- seed-peer
container_name: dragonfly-seed-peer
- seed-client
container_name: client
restart: always
healthcheck:
test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:65000 || exit 1"]
test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=unix:///var/run/dragonfly/dfdaemon.sock || exit 1"]
interval: 1s
timeout: 2s
retries: 30
volumes:
- ./log/peer:/var/log/dragonfly/daemon
- ./config/dfget.yaml:/etc/dragonfly/dfget.yaml:ro
- ./log/client:/var/log/dragonfly
- ./config/client.yaml:/etc/dragonfly/dfdaemon.yaml:ro
ports:
- 65000:65000
- 65001:65001
- 65002:65002
- 4000:4000
- 4001:4001
- 4002:4002

scheduler:
image: dragonflyoss/scheduler:latest
depends_on:
- manager
container_name: dragonfly-scheduler
container_name: scheduler
restart: always
healthcheck:
test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:8002 || exit 1"]
interval: 1s
timeout: 2s
retries: 30
volumes:
- ./log/scheduler:/var/log/dragonfly/scheduler
- ./log/scheduler:/var/log/dragonfly
- ./config/scheduler.yaml:/etc/dragonfly/scheduler.yaml:ro
ports:
- 8002:8002

seed-peer:
image: dragonflyoss/dfdaemon:latest
seed-client:
image: dragonflyoss/client:latest
depends_on:
- manager
- scheduler
container_name: dragonfly-dfdaemon
container_name: seed-client
restart: always
healthcheck:
test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=:65006 || exit 1"]
test: ["CMD-SHELL", "/bin/grpc_health_probe -addr=unix:///var/run/dragonfly/dfdaemon.sock || exit 1"]
interval: 1s
timeout: 2s
retries: 30
volumes:
- ./log/seed-peer:/var/log/dragonfly/daemon
- ./config/seed-peer.yaml:/etc/dragonfly/dfget.yaml:ro
- ./log/seed-client:/var/log/dragonfly
- ./config/seed-client.yaml:/etc/dragonfly/dfdaemon.yaml:ro
ports:
- 65006:65006
- 65007:65007
- 65008:65008
- 4010:4010
- 4011:4011
- 4012:4012
21 changes: 11 additions & 10 deletions deploy/docker-compose/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ set -e

REPO=${REPO:-dragonflyoss}
TAG=${TAG:-latest}
CLIENT_TAG=${CLIENT_TAG:-latest}

DIR=$(cd "$(dirname "$0")" && pwd)
cd $DIR
Expand All @@ -13,8 +14,8 @@ prepare(){

ip=${IP:-$(hostname -i)}

sed "s,__IP__,$ip," template/dfget.template.yaml > config/dfget.yaml
sed "s,__IP__,$ip," template/seed-peer.template.yaml > config/seed-peer.yaml
sed "s,__IP__,$ip," template/client.template.yaml > config/client.yaml
sed "s,__IP__,$ip," template/seed-client.template.yaml > config/seed-client.yaml
sed "s,__IP__,$ip," template/scheduler.template.yaml > config/scheduler.yaml
sed "s,__IP__,$ip," template/manager.template.yaml > config/manager.yaml
}
Expand All @@ -25,7 +26,7 @@ delete_container(){

echo try to clean old containers
${RUNTIME} rm -f dragonfly-redis dragonfly-mysql dragonfly-manager dragonfly-scheduler \
dragonfly-dfdaemon dragonfly-seed-peer
dragonfly-client dragonfly-seed-client
}

run_container(){
Expand All @@ -34,7 +35,7 @@ run_container(){

echo try to clean old containers
${RUNTIME} rm -f dragonfly-redis dragonfly-mysql dragonfly-manager dragonfly-scheduler \
dragonfly-dfdaemon dragonfly-seed-peer
dragonfly-client dragonfly-seed-client

printf "create dragonfly-redis "
${RUNTIME} run -d --name dragonfly-redis --restart=always -p 6379:6379 \
Expand All @@ -55,23 +56,23 @@ run_container(){
-v ${DIR}/config/manager.yaml:/etc/dragonfly/manager.yaml \
${REPO}/manager:${TAG}

printf "create dragonfly-seed-peer "
${RUNTIME} run -d --name dragonfly-seed-peer --restart=always --net=host \
printf "create dragonfly-seed-client "
${RUNTIME} run -d --name dragonfly-seed-client --restart=always --net=host \
-v /tmp/log/dragonfly:/var/log/dragonfly \
-v ${DIR}/config/seed-peer.yaml:/etc/dragonfly/dfget.yaml \
${REPO}/dfdaemon:${TAG}
${REPO}/client:${CLIENT_TAG}

printf "create dragonfly-scheduler "
${RUNTIME} run -d --name dragonfly-scheduler --restart=always --net=host \
-v /tmp/log/dragonfly:/var/log/dragonfly \
-v ${DIR}/config/scheduler.yaml:/etc/dragonfly/scheduler.yaml \
${REPO}/scheduler:${TAG}

printf "create dragonfly-dfdaemon "
${RUNTIME} run -d --name dragonfly-dfdaemon --restart=always --net=host \
printf "create dragonfly-client "
${RUNTIME} run -d --name dragonfly-client --restart=always --net=host \
-v /tmp/log/dragonfly:/var/log/dragonfly \
-v ${DIR}/config/dfget.yaml:/etc/dragonfly/dfget.yaml \
${REPO}/dfdaemon:${TAG}
${REPO}/client:${CLIENT_TAG}
}

prepare
Expand Down
164 changes: 164 additions & 0 deletions deploy/docker-compose/template/client.template.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
# verbose prints log to stdout.
verbose: true

log:
# Specify the logging level [trace, debug, info, warn, error]
level: info

# host is the host configuration for dfdaemon.
host:
## idc is the idc of the host.
idc: ''
## location is the location of the host.
location: ''
## hostname is the hostname of the host.
# hostname: ""
## ip is the advertise ip of the host.
ip: __IP__

server:
# pluginDir is the directory to store plugins.
pluginDir: /var/lib/dragonfly/plugins/dfdaemon/
# cacheDir is the directory to store cache files.
cacheDir: /var/cache/dragonfly/dfdaemon/

download:
server:
# socketPath is the unix socket path for dfdaemon GRPC service.
socketPath: /var/run/dragonfly/dfdaemon.sock
# rateLimit is the default rate limit of the download speed in KiB/MiB/GiB per second, default is 10GiB/s.
rateLimit: 10GiB
# pieceTimeout is the timeout for downloading a piece from source.
pieceTimeout: 30s
# concurrentPieceCount is the number of concurrent pieces to download.
concurrentPieceCount: 10

upload:
server:
# port is the port to the grpc server.
port: 4000
## ip is the listen ip of the grpc server.
# ip: ""
# disableShared indicates whether disable to share data for other peers.
disableShared: false
# rateLimit is the default rate limit of the upload speed in KiB/MiB/GiB per second, default is 10GiB/s.
rateLimit: 10GiB

manager:
# addrs is manager addresses.
addrs:
- http://__IP__:65003

scheduler:
# announceInterval is the interval to announce peer to the scheduler.
# Announcer will provide the scheduler with peer information for scheduling,
# peer information includes cpu, memory, etc.
announceInterval: 10s
# scheduleTimeout is the timeout for scheduling. If the scheduling timesout, dfdaemon will back-to-source
# download if enableBackToSource is true, otherwise dfdaemon will return download failed.
scheduleTimeout: 30s
# maxScheduleCount is the max count of schedule.
maxScheduleCount: 5
# enableBackToSource indicates whether enable back-to-source download, when the scheduling failed.
enableBackToSource: true

dynconfig:
# refreshInterval is the interval to refresh dynamic configuration from manager.
refreshInterval: 1m

storage:
# dir is the directory to store task's metadata and content.
dir: /var/lib/dragonfly/
# keep indicates whether keep the task's metadata and content when the dfdaemon restarts.
keep: true
# writeBufferSize is the buffer size for writing piece to disk, default is 128KB.
writeBufferSize: 131072
# readBufferSize is the buffer size for reading piece from disk, default is 128KB.
readBufferSize: 131072

gc:
# interval is the interval to do gc.
interval: 900s
policy:
# taskTTL is the ttl of the task.
taskTTL: 21600s
# distHighThresholdPercent is the high threshold percent of the disk usage.
# If the disk usage is greater than the threshold, dfdaemon will do gc.
distHighThresholdPercent: 80
# distLowThresholdPercent is the low threshold percent of the disk usage.
# If the disk usage is less than the threshold, dfdaemon will stop gc.
distLowThresholdPercent: 60

proxy:
server:
# port is the port to the proxy server.
port: 4001
## ip is the listen ip of the proxy server.
# ip: ""
## caCert is the root CA cert path with PEM format for the proxy server to generate the server cert.
## If ca_cert is empty, proxy will generate a smaple CA cert by rcgen::generate_simple_self_signed.
## When client requests via the proxy, the client should not verify the server cert and set
## insecure to true. If ca_cert is not empty, proxy will sign the server cert with the CA cert. If openssl is installed,
## you can use openssl to generate the root CA cert and make the system trust the root CA cert.
## Then set the ca_cert and ca_key to the root CA cert and key path. Dfdaemon generates the server cert
## and key, and signs the server cert with the root CA cert. When client requests via the proxy,
## the proxy can intercept the request by the server cert.
# caCert: ""
## caKey is the root CA key path with PEM format for the proxy server to generate the server cert.
## If ca_key is empty, proxy will generate a smaple CA key by rcgen::generate_simple_self_signed.
## When client requests via the proxy, the client should not verify the server cert and set
## insecure to true. If ca_key is not empty, proxy will sign the server cert with the CA cert. If openssl is installed,
## you can use openssl to generate the root CA cert and make the system trust the root CA cert.
## Then set the ca_cert and ca_key to the root CA cert and key path. Dfdaemon generates the server cert
## and key, and signs the server cert with the root CA cert. When client requests via the proxy,
## the proxy can intercept the request by the server cert.
# caKey: ""
# rules is the list of rules for the proxy server.
# regex is the regex of the request url.
# useTLS indicates whether use tls for the proxy backend.
# redirect is the redirect url.
# filteredQueryParams is the filtered query params to generate the task id.
# When filter is ["Signature", "Expires", "ns"], for example:
# http://example.com/xyz?Expires=e1&Signature=s1&ns=docker.io and http://example.com/xyz?Expires=e2&Signature=s2&ns=docker.io
# will generate the same task id.
# Default value includes the filtered query params of s3, gcs, oss, obs, cos.
# `X-Dragonfly-Use-P2P` header can instead of the regular expression of the rule. If the value is "true",
# the request will use P2P technology to distribute the content. If the value is "false",
# but url matches the regular expression in rules. The request will also use P2P technology to distribute the content.
rules:
- regex: 'blobs/sha256.*'
# useTLS: false
# redirect: ""
# filteredQueryParams: []
registryMirror:
# addr is the default address of the registry mirror. Proxy will start a registry mirror service for the
# client to pull the image. The client can use the default address of the registry mirror in
# configuration to pull the image. The `X-Dragonfly-Registry` header can instead of the default address
# of registry mirror.
addr: https://index.docker.io
## certs is the client certs path with PEM format for the registry.
## If registry use self-signed cert, the client should set the
## cert for the registry mirror.
# certs: ""
# disableBackToSource indicates whether disable to download back-to-source when download failed.
disableBackToSource: false
# prefetch pre-downloads full of the task when download with range request.
prefetch: false
# readBufferSize is the buffer size for reading piece from disk, default is 32KB.
readBufferSize: 32768

security:
# enable indicates whether enable security.
enable: false

metrics:
server:
# port is the port to the metrics server.
port: 4002
## ip is the listen ip of the metrics server.
# ip: ""

## tracing is the tracing configuration for dfdaemon.
# tracing:
## addr is the address to report tracing log.
# addr: ""
Loading

0 comments on commit ea850f7

Please sign in to comment.