From 959ddc6352e81de1f12ee0898a9240dbc644b1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Sun, 19 Apr 2020 21:50:46 +0200 Subject: [PATCH] minimal tus implmentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- cmd/reva/upload.go | 49 ++- go.mod | 9 +- go.sum | 92 +++-- .../storageprovider/storageprovider.go | 21 +- .../services/dataprovider/dataprovider.go | 126 ++++-- pkg/eosclient/eosclient.go | 18 +- pkg/storage/fs/eos/eos.go | 23 +- pkg/storage/fs/eos/upload.go | 321 +++++++++++++++ pkg/storage/fs/local/local.go | 37 +- pkg/storage/fs/local/upload.go | 290 ++++++++++++++ pkg/storage/fs/owncloud/owncloud.go | 52 +-- pkg/storage/fs/owncloud/owncloud_unix.go | 2 + pkg/storage/fs/owncloud/upload.go | 374 ++++++++++++++++++ pkg/storage/fs/s3/upload.go | 31 ++ pkg/storage/storage.go | 1 + 15 files changed, 1265 insertions(+), 181 deletions(-) create mode 100644 pkg/storage/fs/eos/upload.go create mode 100644 pkg/storage/fs/local/upload.go create mode 100644 pkg/storage/fs/owncloud/upload.go create mode 100644 pkg/storage/fs/s3/upload.go diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go index d6a3d1c9ab..4d75edab30 100644 --- a/cmd/reva/upload.go +++ b/cmd/reva/upload.go @@ -22,13 +22,16 @@ import ( "fmt" "io" "math" - "net/http" "os" + "path/filepath" "github.com/cheggaaa/pb" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/eventials/go-tus" + "github.com/eventials/go-tus/memorystore" + // TODO(labkode): this should not come from this package. "github.com/cs3org/reva/internal/grpc/services/storageprovider" "github.com/cs3org/reva/pkg/crypto" @@ -65,7 +68,7 @@ func uploadCommand() *command { fmt.Printf("Local file size: %d bytes\n", md.Size()) - client, err := getClient() + gwc, err := getClient() if err != nil { return err } @@ -78,7 +81,7 @@ func uploadCommand() *command { }, } - res, err := client.InitiateFileUpload(ctx, req) + res, err := gwc.InitiateFileUpload(ctx, req) if err != nil { return err } @@ -113,31 +116,43 @@ func uploadCommand() *command { bar.Start() reader := bar.NewProxyReader(fd) - httpReq, err := rhttp.NewRequest(ctx, "PUT", dataServerURL, reader) + // create the tus client. + c := tus.DefaultConfig() + c.Resume = true + c.HttpClient = rhttp.GetHTTPClient(ctx) + c.Store, err = memorystore.NewMemoryStore() if err != nil { return err } + c.Header.Add("X-Reva-Transfer", res.Token) + tusc, err := tus.NewClient(dataServerURL, c) + if err != nil { + return err + } + + metadata := map[string]string{ + "filename": filepath.Base(target), + "dir": filepath.Dir(target), + "checksum": fmt.Sprintf("%s %s", storageprovider.GRPC2PKGXS(xsType).String(), xs), + } + + fingerprint := fmt.Sprintf("%s-%d-%s-%s", md.Name(), md.Size(), md.ModTime(), xs) - httpReq.Header.Set("X-Reva-Transfer", res.Token) - q := httpReq.URL.Query() - q.Add("xs", xs) - q.Add("xs_type", storageprovider.GRPC2PKGXS(xsType).String()) - httpReq.URL.RawQuery = q.Encode() + // create an upload from a file. + upload := tus.NewUpload(reader, md.Size(), metadata, fingerprint) - httpClient := rhttp.GetHTTPClient(ctx) + // create the uploader. + c.Store.Set(upload.Fingerprint, dataServerURL) + uploader := tus.NewUploader(tusc, dataServerURL, upload, 0) - httpRes, err := httpClient.Do(httpReq) + // start the uploading process. + err = uploader.Upload() if err != nil { return err } - defer httpRes.Body.Close() bar.Finish() - if httpRes.StatusCode != http.StatusOK { - return err - } - req2 := &provider.StatRequest{ Ref: &provider.Reference{ Spec: &provider.Reference_Path{ @@ -145,7 +160,7 @@ func uploadCommand() *command { }, }, } - res2, err := client.Stat(ctx, req2) + res2, err := gwc.Stat(ctx, req2) if err != nil { return err } diff --git a/go.mod b/go.mod index b6d6c5a617..da11ac150b 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,14 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/go-cs3apis v0.0.0-20200408065125-6e23f3ecec0a github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/eventials/go-tus v0.0.0-20190617130015-9db47421f6a0 github.com/fatih/color v1.7.0 // indirect github.com/go-openapi/strfmt v0.19.2 // indirect github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.3.5 github.com/gomodule/redigo v2.0.0+incompatible github.com/google/uuid v1.1.1 - github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 + github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/huandu/xstrings v1.3.0 // indirect github.com/imdario/mergo v0.3.8 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible @@ -31,12 +32,12 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect - github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect github.com/rs/cors v1.7.0 github.com/rs/zerolog v1.18.0 + github.com/tus/tusd v1.1.0 go.opencensus.io v0.22.3 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a - golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 google.golang.org/grpc v1.28.1 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect @@ -45,3 +46,5 @@ require ( ) go 1.13 + +replace github.com/eventials/go-tus => github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a diff --git a/go.sum b/go.sum index a755afea1d..f3b539bb10 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro= contrib.go.opencensus.io/exporter/jaeger v0.2.0 h1:nhTv/Ry3lGmqbJ/JGvCjWxBl5ozRfqo86Ngz59UAlfk= contrib.go.opencensus.io/exporter/jaeger v0.2.0/go.mod h1:ukdzwIYYHgZ7QYtwVFQUjiT28BJHiMhTERo32s6qVgM= contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= @@ -16,22 +18,19 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a h1:6tD4saJb8wmYF6Llz96ZJwUQ5r2GyTBFA2VEB5z8gVY= +github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a/go.mod h1:XYuK1S5+kS6FGhlIUFuZFPvWiSrOIoLk6+ro33Xce3Y= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.30.1 h1:cUMxtoFvIHhScZgv17tGxw15r6rVKJHR1hsIFRx9hcA= -github.com/aws/aws-sdk-go v1.30.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.2 h1:0vuroAsbPwVbP91MMaUmFLnrQcFBhmjQnnXaH1kcnPw= -github.com/aws/aws-sdk-go v1.30.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.3 h1:tmaR+qpBSig6RfhP9IoxALJEE1m0vfLy5tlnEIXu6WI= -github.com/aws/aws-sdk-go v1.30.3/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.30.4 h1:dpQgypC3rld2Uuz+/2u+0nbfmmyEWxau6v1hdAlvoc8= -github.com/aws/aws-sdk-go v1.30.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.20.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY= github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98= github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= @@ -40,10 +39,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cs3org/go-cs3apis v0.0.0-20200324115356-e04b4fd75f03 h1:JGANezYNs/VM1Mpqu/noYqAT8HzsheysyArEaz4OnwM= -github.com/cs3org/go-cs3apis v0.0.0-20200324115356-e04b4fd75f03/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/go-cs3apis v0.0.0-20200403073321-0519c6823b48 h1:iR2JGsyZwRg6N1k9QuCJhXRH4CQUnCZNoJ0VmnjF2xE= -github.com/cs3org/go-cs3apis v0.0.0-20200403073321-0519c6823b48/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20200408065125-6e23f3ecec0a h1:+enFdliTCV/aaLAvLmeka/r9wUoEypngV4JD5Gy92Uc= github.com/cs3org/go-cs3apis v0.0.0-20200408065125-6e23f3ecec0a/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,6 +53,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= @@ -70,12 +66,15 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -86,32 +85,44 @@ github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181004151105-1babbf986f6f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.0 h1:gvV6jG9dTgFEncxo+AF7PH6MZXi/vZl25owA/8Dg8Wo= github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -131,12 +142,19 @@ github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/moul/http2curl v0.0.0-20170919181001-9ac6cf4d929b/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY= github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/ory/fosite v0.31.0 h1:NZ0FA4ywPEYrCGLNVBAz2dq8vTacLDbbO4Iiy68WCKQ= github.com/ory/fosite v0.31.0/go.mod h1:lSSqjo8Kr/U1P3kJWxsNGHmq7TnH/7pS1ijvQRT7G+g= github.com/ory/go-convenience v0.1.0 h1:zouLKfF2GoSGnJwGq+PE/nJAE6dj2Zj5QlTgmMTsTS8= @@ -158,29 +176,27 @@ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= +github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -194,16 +210,23 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tus/tusd v1.1.0 h1:y2oBFGeOyqlGgyqD0CloH8FuBrjDk0Tq1IQWvAZnyG8= +github.com/tus/tusd v1.1.0/go.mod h1:3DWPOdeCnjBwKtv98y5dSws3itPqfce5TVa0s59LRiA= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/vimeo/go-util v1.2.0/go.mod h1:s13SMDTSO7AjH1nbgp707mfN5JFIWUFDU5MDDuRRtKs= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= @@ -213,15 +236,17 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -231,6 +256,7 @@ golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced h1:4oqSq7eft7MdPKBGQK11X9 golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -239,40 +265,51 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181021155630-eda9bb28ed51/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190415081028-16da32be82c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd h1:r7DufRZuZbWB7j439YfAzP8RPDa9unLkpwQKUYbIMPI= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 h1:xtNn7qFlagY2mQNFHMSRPjT2RkOV4OXM7P5TVy9xATo= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb h1:i1Ppqkc3WQXikh8bXiwHqAN5Rv3/qDCcRk0/Otx73BY= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -282,10 +319,9 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k= google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +gopkg.in/Acconut/lockfile.v1 v1.1.0/go.mod h1:6UCz3wJ8tSFUsPR6uP/j8uegEtDuEEqFxlpi0JI4Umw= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= @@ -294,15 +330,21 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU= gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/square/go-jose.v2 v2.1.9/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index e767d3a99d..a7ddfb4e3b 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -24,6 +24,7 @@ import ( "net/url" "os" "path" + "strconv" "strings" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -243,14 +244,30 @@ func (s *service) InitiateFileDownload(ctx context.Context, req *provider.Initia func (s *service) InitiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*provider.InitiateFileUploadResponse, error) { // TODO(labkode): same considerations as download log := appctx.GetLogger(ctx) - url := *s.dataServerURL newRef, err := s.unwrap(ctx, req.Ref) if err != nil { return &provider.InitiateFileUploadResponse{ Status: status.NewInternal(ctx, err, "error unwrapping path"), }, nil } - url.Path = path.Join("/", url.Path, newRef.GetPath()) + var uploadLength int64 + if req.Opaque != nil && req.Opaque.Map != nil && req.Opaque.Map["Upload-Length"] != nil { + var err error + uploadLength, err = strconv.ParseInt(string(req.Opaque.Map["Upload-Length"].Value), 10, 64) + if err != nil { + return &provider.InitiateFileUploadResponse{ + Status: status.NewInternal(ctx, err, "error parsing upload length"), + }, nil + } + } + uploadID, err := s.storage.InitiateUpload(ctx, newRef, uploadLength) + if err != nil { + return &provider.InitiateFileUploadResponse{ + Status: status.NewInternal(ctx, err, "error getting upload id"), + }, nil + } + url := *s.dataServerURL + url.Path = path.Join("/", url.Path, uploadID) log.Info().Str("data-server", url.String()). Str("fn", req.Ref.GetPath()). Str("xs", fmt.Sprintf("%+v", s.conf.AvailableXS)). diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 831aa23622..b373133ae9 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -21,13 +21,13 @@ package dataprovider import ( "fmt" "net/http" - "os" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" ) func init() { @@ -35,10 +35,9 @@ func init() { } type config struct { - Prefix string `mapstructure:"prefix"` - Driver string `mapstructure:"driver"` - TmpFolder string `mapstructure:"tmp_folder"` - Drivers map[string]map[string]interface{} `mapstructure:"drivers"` + Prefix string `mapstructure:"prefix"` + Driver string `mapstructure:"driver"` + Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } type svc struct { @@ -54,18 +53,6 @@ func New(m map[string]interface{}) (global.Service, error) { return nil, err } - if conf.Prefix == "" { - conf.Prefix = "data" - } - - if conf.TmpFolder == "" { - conf.TmpFolder = os.TempDir() - } - - if err := os.MkdirAll(conf.TmpFolder, 0755); err != nil { - return nil, errors.Wrap(err, "could not create tmp dir") - } - fs, err := getFS(conf) if err != nil { return nil, err @@ -75,8 +62,8 @@ func New(m map[string]interface{}) (global.Service, error) { storage: fs, conf: conf, } - s.setHandler() - return s, nil + err = s.setHandler() + return s, err } // Close performs cleanup. @@ -88,6 +75,12 @@ func (s *svc) Unprotected() []string { return []string{} } +// Create a new DataStore instance which is responsible for +// storing the uploaded file on disk in the specified directory. +// This path _must_ exist before tusd will store uploads in it. +// If you want to save them on a different medium, for example +// a remote FTP server, you can implement your own storage backend +// by implementing the tusd.DataStore interface. func getFS(c *config) (storage.FS, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { return f(c.Drivers[c.Driver]) @@ -103,24 +96,83 @@ func (s *svc) Handler() http.Handler { return s.handler } -func (s *svc) setHandler() { - s.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "HEAD": - addCorsHeader(w) - w.WriteHeader(http.StatusOK) - return - case "GET": - s.doGet(w, r) - return - case "PUT": - s.doPut(w, r) - return - default: - w.WriteHeader(http.StatusNotImplemented) - return +// Composable is the interface that a struct needs to implement to be composable by this composer +type Composable interface { + UseIn(composer *tusd.StoreComposer) +} + +func (s *svc) setHandler() (err error) { + composable, ok := s.storage.(Composable) + if ok { + // A storage backend for tusd may consist of multiple different parts which + // handle upload creation, locking, termination and so on. The composer is a + // place where all those separated pieces are joined together. In this example + // we only use the file store but you may plug in multiple. + composer := tusd.NewStoreComposer() + + // let the composable storage tell tus which extensions it supports + composable.UseIn(composer) + + config := tusd.Config{ + BasePath: s.conf.Prefix, + StoreComposer: composer, + //Logger: logger, // TODO use logger + } + + handler, err := tusd.NewUnroutedHandler(config) + if err != nil { + return err } - }) + + s.handler = handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + log := appctx.GetLogger(r.Context()) + log.Info().Msgf("tusd routing: path=%s", r.URL.Path) + + switch r.Method { + // old fashioned download. + + // GET is not part of the tus.io protocol + // currently there is no way to GET an upload that is in progress + // TODO allow range based get requests? that end before the current offset + case "GET": + s.doGet(w, r) + + // tus.io based upload + + // uploads are initiated using the CS3 APIs Initiate Download call + case "POST": + handler.PostFile(w, r) + case "HEAD": + handler.HeadFile(w, r) + case "PATCH": + handler.PatchFile(w, r) + // TODO Only attach the DELETE handler if the Terminate() method is provided + case "DELETE": + handler.DelFile(w, r) + } + })) + } else { + s.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "HEAD": + addCorsHeader(w) + w.WriteHeader(http.StatusOK) + return + case "GET": + s.doGet(w, r) + return + case "PUT": + s.doPut(w, r) + return + default: + w.WriteHeader(http.StatusNotImplemented) + return + } + }) + } + + return err } func addCorsHeader(res http.ResponseWriter) { diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index 767a9e2621..c9a0c38395 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -591,12 +591,8 @@ func (c *Client) Read(ctx context.Context, username, path string) (io.ReadCloser return os.Open(localTarget) } -// Write writes a file to the mgm +// Write writes a stream to the mgm func (c *Client) Write(ctx context.Context, username, path string, stream io.ReadCloser) error { - unixUser, err := c.getUnixUser(username) - if err != nil { - return err - } fd, err := ioutil.TempFile(c.opt.CacheDirectory, "eoswrite-") if err != nil { return err @@ -609,8 +605,18 @@ func (c *Client) Write(ctx context.Context, username, path string, stream io.Rea if err != nil { return err } + + return c.WriteFile(ctx, username, path, fd.Name()) +} + +// WriteFile writes an existing file to the mgm +func (c *Client) WriteFile(ctx context.Context, username, path, source string) error { + unixUser, err := c.getUnixUser(username) + if err != nil { + return err + } xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path) - cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", fd.Name(), xrdPath, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s", unixUser.Uid, unixUser.Gid)) + cmd := exec.CommandContext(ctx, c.opt.XrdcopyBinary, "--nopbar", "--silent", "-f", source, xrdPath, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s", unixUser.Uid, unixUser.Gid)) _, _, err = c.execute(ctx, cmd) return err } diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index 3f8c56e7e0..a4e8633ee5 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -79,6 +79,9 @@ type config struct { // ShadowNamespace for storing shadow data ShadowNamespace string `mapstructure:"shadow_namespace"` + // UploadsNamespace for storing upload data + UploadsNamespace string `mapstructure:"uploads_namespace"` + // ShareFolder defines the name of the folder in the // shadowed namespace. Ex: /eos/user/.shadow/h/hugo/MyShares ShareFolder string `mapstructure:"share_folder"` @@ -1130,26 +1133,6 @@ func (fs *eosfs) Download(ctx context.Context, ref *provider.Reference) (io.Read return fs.c.Read(ctx, u.Username, fn) } -func (fs *eosfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - u, err := getUser(ctx) - if err != nil { - return errors.Wrap(err, "eos: no user in ctx") - } - - p, err := fs.resolve(ctx, u, ref) - if err != nil { - return errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return errtypes.PermissionDenied("eos: cannot download under the virtual share folder") - } - - fn := fs.wrap(ctx, p) - - return fs.c.Write(ctx, u.Username, fn, r) -} - func (fs *eosfs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { u, err := getUser(ctx) if err != nil { diff --git a/pkg/storage/fs/eos/upload.go b/pkg/storage/fs/eos/upload.go new file mode 100644 index 0000000000..ecc80f7930 --- /dev/null +++ b/pkg/storage/fs/eos/upload.go @@ -0,0 +1,321 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package eos + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus +func (fs *eosfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + + p, err := fs.resolve(ctx, u, ref) + if err != nil { + return errors.Wrap(err, "eos: error resolving reference") + } + + if fs.isShareFolder(ctx, p) { + return errtypes.PermissionDenied("eos: cannot upload under the virtual share folder") + } + + fn := fs.wrap(ctx, p) + + return fs.c.Write(ctx, u.Username, fn, r) +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// TODO read optional content for small files in this request +func (fs *eosfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + u, err := getUser(ctx) + if err != nil { + return "", errors.Wrap(err, "eos: no user in ctx") + } + + np, err := fs.resolve(ctx, u, ref) + if err != nil { + return "", errors.Wrap(err, "eos: error resolving reference") + } + + p := fs.unwrap(ctx, np) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *eosfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) +} + +// NewUpload creates a new upload using the size as the file's length. To determine where to write the binary data +// the Fileinfo metadata must contain a dir and a filename. +// returns a unique id which is used to identify the upload. The properties Size and MetaData will be filled. +func (fs *eosfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("eos: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("eos: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("eos: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + np := fs.wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + + log.Debug().Interface("info", info).Msg("eos: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "eos: error resolving upload path") + } + info.Storage = map[string]string{ + "Type": "EOSStore", + "InternalDestination": np, + } + // Create binary file with no content + + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: binPath + ".info", + fs: fs, + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +// TODO use a subdirectory in the shadow tree +func (fs *eosfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + return filepath.Join(fs.conf.CacheDirectory, uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *eosfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + binPath, err := fs.getUploadPath(ctx, id) + if err != nil { + return nil, err + } + infoPath := binPath + ".info" + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(binPath) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + fs: fs, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *eosfs +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// GetReader returns an io.Readerfor the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +// TODO use the grpc api to directly stream to a temporary uploads location in the eos shadow tree +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() + + return n, err +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + checksum := upload.info.MetaData["checksum"] + if checksum != "" { + // check checksum + s := strings.SplitN(checksum, " ", 2) + if len(s) == 2 { + alg, hash := s[0], s[1] + + log := appctx.GetLogger(ctx) + log.Debug(). + Interface("info", upload.info). + Str("alg", alg). + Str("hash", hash). + Msg("eos: TODO check checksum") // TODO this is done by eos if we write chunks to it directly + + } + } + np := upload.info.Storage["InternalDestination"] + + // TODO check etag with If-Match header + // if destination exists + //if _, err := os.Stat(np); err == nil { + // copy attributes of existing file to tmp file befor overwriting the target? + // eos creates revisions internally + //} + + u, err := getUser(ctx) + if err != nil { + return errors.Wrap(err, "eos: no user in ctx") + } + err = upload.fs.c.WriteFile(ctx, u.Username, np, upload.binPath) + + // only delete the upload if it was successfully written to eos + if err == nil { + // cleanup in the background, delete might take a while and we don't need to wait for it to finish + go func() { + if err := os.Remove(upload.infoPath); err != nil { + log := appctx.GetLogger(ctx) + log.Err(err).Interface("info", upload.info).Msg("eos: could not delete upload info") + } + if err := os.Remove(upload.binPath); err != nil { + log := appctx.GetLogger(ctx) + log.Err(err).Interface("info", upload.info).Msg("eos: could not delete upload binary") + } + }() + } + // metadata propagation is left to the storage implementation + return err +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returnsa a TerminatableUpload +func (fs *eosfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + return err + } + if err := os.Remove(upload.binPath); err != nil { + return err + } + return nil +} diff --git a/pkg/storage/fs/local/local.go b/pkg/storage/fs/local/local.go index 573dffdcc8..ad03253cc2 100644 --- a/pkg/storage/fs/local/local.go +++ b/pkg/storage/fs/local/local.go @@ -49,6 +49,8 @@ type config struct { Root string `mapstructure:"root"` EnableHome bool `mapstructure:"enable_home"` UserLayout string `mapstructure:"user_layout"` + // Uploads fsolder should be on the same partition as root to make the final rename not fall back to a copy and delete + Uploads string `mapstructure:"uploads"` } func parseConfig(m map[string]interface{}) (*config, error) { @@ -82,6 +84,14 @@ func New(m map[string]interface{}) (storage.FS, error) { return nil, errors.Wrap(err, "local: could not create namespace dir") } + if c.Uploads == "" { + c.Uploads = path.Join(c.Root, ".uploads") + } + + if err := os.MkdirAll(c.Uploads, 0700); err != nil { + return nil, errors.Wrap(err, "could not create uploads dir "+c.Uploads) + } + return &localfs{root: c.Root, conf: c}, nil } @@ -335,33 +345,6 @@ func (fs *localfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]* return finfos, nil } -func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - fn, err := fs.resolve(ctx, ref) - if err != nil { - return errors.Wrap(err, "error resolving ref") - } - - // we cannot rely on /tmp as it can live in another partition and we can - // hit invalid cross-device link errors, so we create the tmp file in the same directory - // the file is supposed to be written. - tmp, err := ioutil.TempFile(path.Dir(fn), "._reva_atomic_upload") - if err != nil { - return errors.Wrap(err, "localfs: error creating tmp fn at "+path.Dir(fn)) - } - - _, err = io.Copy(tmp, r) - if err != nil { - return errors.Wrap(err, "localfs: eror writing to tmp file "+tmp.Name()) - } - - // TODO(labkode): make sure rename is atomic, missing fsync ... - if err := os.Rename(tmp.Name(), fn); err != nil { - return errors.Wrap(err, "localfs: error renaming from "+tmp.Name()+" to "+fn) - } - - return nil -} - func (fs *localfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { fn, err := fs.resolve(ctx, ref) if err != nil { diff --git a/pkg/storage/fs/local/upload.go b/pkg/storage/fs/local/upload.go new file mode 100644 index 0000000000..ace1c4b5ef --- /dev/null +++ b/pkg/storage/fs/local/upload.go @@ -0,0 +1,290 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package local + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus +func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + fn, err := fs.resolve(ctx, ref) + if err != nil { + return errors.Wrap(err, "error resolving ref") + } + + // we cannot rely on /tmp as it can live in another partition and we can + // hit invalid cross-device link errors, so we create the tmp file in the same directory + // the file is supposed to be written. + tmp, err := ioutil.TempFile(filepath.Dir(fn), "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "localfs: error creating tmp fn at "+filepath.Dir(fn)) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "localfs: eror writing to tmp file "+tmp.Name()) + } + + // TODO(labkode): make sure rename is atomic, missing fsync ... + if err := os.Rename(tmp.Name(), fn); err != nil { + return errors.Wrap(err, "localfs: error renaming from "+tmp.Name()+" to "+fn) + } + + return nil +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// It resolves the resurce and then reuses the NewUpload function +// Currently requires the uploadLength to be set +// TODO to implement LengthDeferrerDataStore make size optional +// TODO read optional content for small files in this request +func (fs *localfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + np, err := fs.resolve(ctx, ref) + if err != nil { + return "", errors.Wrap(err, "localfs: error resolving reference") + } + + p := fs.unwrap(ctx, np) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *localfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) + // TODO composer.UseConcater(fs) + // TODO composer.UseLengthDeferrer(fs) +} + +// NewUpload creates a new upload using the size as the file's length. To determine where to write the binary data +// the Fileinfo metadata must contain a dir and a filename. +// returns a unique id which is used to identify the upload. The properties Size and MetaData will be filled. +func (fs *localfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("localfs: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("localfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("localfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + np := fs.wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + + log.Debug().Interface("info", info).Msg("localfs: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "localfs: error resolving upload path") + } + info.Storage = map[string]string{ + "Type": "LocalStore", + "InternalDestination": np, + } + // Create binary file with no content + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: binPath + ".info", + fs: fs, + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +func (fs *localfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + return filepath.Join(fs.conf.Uploads, uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *localfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + binPath, err := fs.getUploadPath(ctx, id) + if err != nil { + return nil, err + } + infoPath := binPath + ".info" + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(binPath) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + fs: fs, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *localfs +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// GetReader returns an io.Readerfor the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() + + return n, err +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + np := upload.info.Storage["InternalDestination"] + + // TODO check etag with If-Match header + // if destination exists + //if _, err := os.Stat(np); err == nil { + // the local storage does not store metadata + // the fileid is based on the path, so no we do not need to copy it to the new file + // the local storage does not track revisions + //} + + err := os.Rename(upload.binPath, np) + + // metadata propagation is left to the storage implementation + return err +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returnsa a TerminatableUpload +func (fs *localfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + return err + } + if err := os.Remove(upload.binPath); err != nil { + return err + } + return nil +} diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 0380ac7f0c..9085024a45 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -144,6 +144,7 @@ const ( mdPrefix string = "user.oc.md." // arbitrary metadada favPrefix string = "user.oc.fav." // favorite flag, per user etagPrefix string = "user.oc.etag." // allow overriding a calculated etag with one from the extended attributes + //checksumPrefix string = "user.oc.cs." // TODO add checksum support ) func init() { @@ -877,6 +878,7 @@ func (fs *ocfs) CreateHome(ctx context.Context) error { path.Join(fs.c.DataDirectory, layout, "files"), path.Join(fs.c.DataDirectory, layout, "files_trashbin"), path.Join(fs.c.DataDirectory, layout, "files_versions"), + path.Join(fs.c.DataDirectory, layout, "uploads"), } for _, v := range homePaths { @@ -1240,50 +1242,9 @@ func (fs *ocfs) ListFolder(ctx context.Context, ref *provider.Reference) ([]*pro return finfos, nil } -func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { - np, err := fs.resolve(ctx, ref) - if err != nil { - return errors.Wrap(err, "ocfs: error resolving reference") - } - - // we cannot rely on /tmp as it can live in another partition and we can - // hit invalid cross-device link errors, so we create the tmp file in the same directory - // the file is supposed to be written. - tmp, err := ioutil.TempFile(path.Dir(np), "._reva_atomic_upload") - if err != nil { - return errors.Wrap(err, "ocfs: error creating tmp fn at "+path.Dir(np)) - } - defer os.RemoveAll(tmp.Name()) - - _, err = io.Copy(tmp, r) - tmp.Close() - if err != nil { - return errors.Wrap(err, "ocfs: error writing to tmp file "+tmp.Name()) - } - - // if destination exists - if _, err := os.Stat(np); err == nil { - // copy attributes of existing file to tmp file - if err := fs.copyMD(np, tmp.Name()); err != nil { - return errors.Wrap(err, "ocfs: error copying metadata from "+np+" to "+tmp.Name()) - } - // create revision - if err := fs.archiveRevision(ctx, np); err != nil { - return err - } - } - - // TODO(jfd): make sure rename is atomic, missing fsync ... - if err := os.Rename(tmp.Name(), np); err != nil { - return errors.Wrap(err, "ocfs: error renaming from "+tmp.Name()+" to "+np) - } - - return nil -} - -func (fs *ocfs) archiveRevision(ctx context.Context, np string) error { +func (fs *ocfs) archiveRevision(ctx context.Context, vbp string, np string) error { // move existing file to versions dir - vp := fmt.Sprintf("%s.v%d", fs.getVersionsPath(ctx, np), time.Now().Unix()) + vp := fmt.Sprintf("%s.v%d", vbp, time.Now().Unix()) if err := os.MkdirAll(path.Dir(vp), 0700); err != nil { return errors.Wrap(err, "ocfs: error creating versions dir "+vp) } @@ -1403,7 +1364,7 @@ func (fs *ocfs) RestoreRevision(ctx context.Context, ref *provider.Reference, re defer source.Close() // destination should be available, otherwise we could not have navigated to its revisions - if err := fs.archiveRevision(ctx, np); err != nil { + if err := fs.archiveRevision(ctx, fs.getVersionsPath(ctx, np), np); err != nil { return err } @@ -1558,3 +1519,6 @@ func (fs *ocfs) RestoreRecycleItem(ctx context.Context, key string) error { // TODO(jfd) restore versions return nil } + +// TODO propagate etag and mtime or append event to history? propagate on disk ... +// - but propagation is a separate task. only if upload was successful ... diff --git a/pkg/storage/fs/owncloud/owncloud_unix.go b/pkg/storage/fs/owncloud/owncloud_unix.go index 24760ffdde..26bb3b0cea 100755 --- a/pkg/storage/fs/owncloud/owncloud_unix.go +++ b/pkg/storage/fs/owncloud/owncloud_unix.go @@ -31,6 +31,8 @@ import ( "github.com/cs3org/reva/pkg/appctx" ) +// TODO(jfd) get rid of the differences between unix and windows. the inode and dev should never be used for the etag because it interferes with backups + // calcEtag will create an etag based on the md5 of // - mtime, // - inode (if available), diff --git a/pkg/storage/fs/owncloud/upload.go b/pkg/storage/fs/owncloud/upload.go new file mode 100644 index 0000000000..0d40fbc2df --- /dev/null +++ b/pkg/storage/fs/owncloud/upload.go @@ -0,0 +1,374 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package owncloud + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/user" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus +func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + np, err := fs.resolve(ctx, ref) + if err != nil { + return errors.Wrap(err, "ocfs: error resolving reference") + } + + // we cannot rely on /tmp as it can live in another partition and we can + // hit invalid cross-device link errors, so we create the tmp file in the same directory + // the file is supposed to be written. + tmp, err := ioutil.TempFile(filepath.Dir(np), "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "ocfs: error creating tmp fn at "+filepath.Dir(np)) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "ocfs: error writing to tmp file "+tmp.Name()) + } + + // if destination exists + if _, err := os.Stat(np); err == nil { + // copy attributes of existing file to tmp file + if err := fs.copyMD(np, tmp.Name()); err != nil { + return errors.Wrap(err, "ocfs: error copying metadata from "+np+" to "+tmp.Name()) + } + // create revision + if err := fs.archiveRevision(ctx, fs.getVersionsPath(ctx, np), np); err != nil { + return err + } + } + + // TODO(jfd): make sure rename is atomic, missing fsync ... + if err := os.Rename(tmp.Name(), np); err != nil { + return errors.Wrap(err, "ocfs: error renaming from "+tmp.Name()+" to "+np) + } + + return nil +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// TODO read optional content for small files in this request +func (fs *ocfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + np, err := fs.resolve(ctx, ref) + if err != nil { + return "", errors.Wrap(err, "ocfs: error resolving reference") + } + + p := fs.unwrap(ctx, np) + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(p), + "dir": filepath.Dir(p), + }, + Size: uploadLength, + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *ocfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) + composer.UseConcater(fs) + composer.UseLengthDeferrer(fs) +} + +// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol +// - the storage needs to implement NewUpload and GetUpload +// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload + +func (fs *ocfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("ocfs: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("ocfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("ocfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + np := fs.wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + + log.Debug().Interface("info", info).Msg("ocfs: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "ocfs: error resolving upload path") + } + info.Storage = map[string]string{ + "Type": "OwnCloudStore", + "InternalDestination": np, + } + // Create binary file in the upload folder with no content + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: binPath + ".info", + fs: fs, + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +func (fs *ocfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + u, ok := user.ContextGetUser(ctx) + if !ok { + err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + return "", err + } + return filepath.Join(fs.c.DataDirectory, u.Username, "uploads", uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *ocfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + binPath, err := fs.getUploadPath(ctx, id) + if err != nil { + return nil, err + } + infoPath := binPath + ".info" + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(binPath) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + fs: fs, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *ocfs +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk + + return n, err +} + +// GetReader returns an io.Readerfor the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) error { + + /* + checksum := upload.info.MetaData["checksum"] + if checksum != "" { + // TODO check checksum + s := strings.SplitN(checksum, " ", 2) + if len(s) == 2 { + alg, hash := s[0], s[1] + + } + } + */ + + np := upload.info.Storage["InternalDestination"] + + // if destination exists + // TODO check etag with If-Match header + if _, err := os.Stat(np); err == nil { + // copy attributes of existing file to tmp file + if err := upload.fs.copyMD(np, upload.binPath); err != nil { + return errors.Wrap(err, "ocfs: error copying metadata from "+np+" to "+upload.binPath) + } + // create revision + if err := upload.fs.archiveRevision(ctx, upload.fs.getVersionsPath(ctx, np), np); err != nil { + return err + } + } + + err := os.Rename(upload.binPath, np) + // TODO else remove info? or leave as history? + + // metadata propagation is left to the storage implementation + return err +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returnsa a TerminatableUpload +func (fs *ocfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + return err + } + if err := os.Remove(upload.binPath); err != nil { + return err + } + return nil +} + +// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation +// - the storage needs to implement AsLengthDeclarableUpload +// - the upload needs to implement DeclareLength + +// AsLengthDeclarableUpload returnsa a LengthDeclarableUpload +func (fs *ocfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload { + return upload.(*fileUpload) +} + +// DeclareLength updates the upload length information +func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { + upload.info.Size = length + upload.info.SizeIsDeferred = false + return upload.writeInfo() +} + +// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation +// - the storage needs to implement AsConcatableUpload +// - the upload needs to implement ConcatUploads + +// AsConcatableUpload returnsa a ConcatableUpload +func (fs *ocfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload { + return upload.(*fileUpload) +} + +// ConcatUploads concatenates multiple uploads +func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*fileUpload) + + src, err := os.Open(fileUpload.binPath) + if err != nil { + return err + } + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + + return +} diff --git a/pkg/storage/fs/s3/upload.go b/pkg/storage/fs/s3/upload.go new file mode 100644 index 0000000000..a87a96d7ff --- /dev/null +++ b/pkg/storage/fs/s3/upload.go @@ -0,0 +1,31 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package s3 + +import ( + "context" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" +) + +// InitiateUpload returns an upload id that can be used for uploads with tus +func (fs *s3FS) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (uploadID string, err error) { + return "", errtypes.NotSupported("op not supported") +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 3457c0c552..776af7a17b 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -36,6 +36,7 @@ type FS interface { Move(ctx context.Context, oldRef, newRef *provider.Reference) error GetMD(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) ListFolder(ctx context.Context, ref *provider.Reference) ([]*provider.ResourceInfo, error) + InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64) (string, error) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error)