diff --git a/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.cc b/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.cc index 47ae1eae9..4fad4d230 100644 --- a/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.cc +++ b/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.cc @@ -237,6 +237,7 @@ const ::google::protobuf::uint32 TableStruct_flyteidl_2fservice_2fdataproxy_2epr PROTOBUF_FIELD_OFFSET(::flyteidl::service::CreateUploadLocationRequest, filename_), PROTOBUF_FIELD_OFFSET(::flyteidl::service::CreateUploadLocationRequest, expires_in_), PROTOBUF_FIELD_OFFSET(::flyteidl::service::CreateUploadLocationRequest, content_md5_), + PROTOBUF_FIELD_OFFSET(::flyteidl::service::CreateUploadLocationRequest, filename_root_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::flyteidl::service::CreateDownloadLocationRequest, _internal_metadata_), ~0u, // no _extensions_ @@ -294,13 +295,13 @@ const ::google::protobuf::uint32 TableStruct_flyteidl_2fservice_2fdataproxy_2epr static const ::google::protobuf::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::flyteidl::service::CreateUploadLocationResponse)}, { 8, -1, sizeof(::flyteidl::service::CreateUploadLocationRequest)}, - { 18, -1, sizeof(::flyteidl::service::CreateDownloadLocationRequest)}, - { 25, -1, sizeof(::flyteidl::service::CreateDownloadLocationResponse)}, - { 32, -1, sizeof(::flyteidl::service::CreateDownloadLinkRequest)}, - { 41, -1, sizeof(::flyteidl::service::CreateDownloadLinkResponse)}, - { 49, -1, sizeof(::flyteidl::service::PreSignedURLs)}, - { 56, -1, sizeof(::flyteidl::service::GetDataRequest)}, - { 62, -1, sizeof(::flyteidl::service::GetDataResponse)}, + { 19, -1, sizeof(::flyteidl::service::CreateDownloadLocationRequest)}, + { 26, -1, sizeof(::flyteidl::service::CreateDownloadLocationResponse)}, + { 33, -1, sizeof(::flyteidl::service::CreateDownloadLinkRequest)}, + { 42, -1, sizeof(::flyteidl::service::CreateDownloadLinkResponse)}, + { 50, -1, sizeof(::flyteidl::service::PreSignedURLs)}, + { 57, -1, sizeof(::flyteidl::service::GetDataRequest)}, + { 63, -1, sizeof(::flyteidl::service::GetDataResponse)}, }; static ::google::protobuf::Message const * const file_default_instances[] = { @@ -330,56 +331,57 @@ const char descriptor_table_protodef_flyteidl_2fservice_2fdataproxy_2eproto[] = "als.proto\"v\n\034CreateUploadLocationRespons" "e\022\022\n\nsigned_url\030\001 \001(\t\022\022\n\nnative_url\030\002 \001(" "\t\022.\n\nexpires_at\030\003 \001(\0132\032.google.protobuf." - "Timestamp\"\224\001\n\033CreateUploadLocationReques" + "Timestamp\"\253\001\n\033CreateUploadLocationReques" "t\022\017\n\007project\030\001 \001(\t\022\016\n\006domain\030\002 \001(\t\022\020\n\010fi" "lename\030\003 \001(\t\022-\n\nexpires_in\030\004 \001(\0132\031.googl" "e.protobuf.Duration\022\023\n\013content_md5\030\005 \001(\014" - "\"f\n\035CreateDownloadLocationRequest\022\022\n\nnat" - "ive_url\030\001 \001(\t\022-\n\nexpires_in\030\002 \001(\0132\031.goog" - "le.protobuf.Duration:\002\030\001\"h\n\036CreateDownlo" - "adLocationResponse\022\022\n\nsigned_url\030\001 \001(\t\022." - "\n\nexpires_at\030\002 \001(\0132\032.google.protobuf.Tim" - "estamp:\002\030\001\"\320\001\n\031CreateDownloadLinkRequest" - "\0225\n\rartifact_type\030\001 \001(\0162\036.flyteidl.servi" - "ce.ArtifactType\022-\n\nexpires_in\030\002 \001(\0132\031.go" - "ogle.protobuf.Duration\022C\n\021node_execution" - "_id\030\003 \001(\0132&.flyteidl.core.NodeExecutionI" - "dentifierH\000B\010\n\006source\"\242\001\n\032CreateDownload" - "LinkResponse\022\026\n\nsigned_url\030\001 \003(\tB\002\030\001\0222\n\n" - "expires_at\030\002 \001(\0132\032.google.protobuf.Times" - "tampB\002\030\001\0228\n\017pre_signed_urls\030\003 \001(\0132\037.flyt" - "eidl.service.PreSignedURLs\"S\n\rPreSignedU" - "RLs\022\022\n\nsigned_url\030\001 \003(\t\022.\n\nexpires_at\030\002 " - "\001(\0132\032.google.protobuf.Timestamp\"#\n\016GetDa" - "taRequest\022\021\n\tflyte_url\030\001 \001(\t\"\262\001\n\017GetData" - "Response\0220\n\013literal_map\030\001 \001(\0132\031.flyteidl" - ".core.LiteralMapH\000\022:\n\017pre_signed_urls\030\002 " - "\001(\0132\037.flyteidl.service.PreSignedURLsH\000\022)" - "\n\007literal\030\003 \001(\0132\026.flyteidl.core.LiteralH" - "\000B\006\n\004data*C\n\014ArtifactType\022\033\n\027ARTIFACT_TY" - "PE_UNDEFINED\020\000\022\026\n\022ARTIFACT_TYPE_DECK\020\0012\342" - "\004\n\020DataProxyService\022\240\001\n\024CreateUploadLoca" - "tion\022-.flyteidl.service.CreateUploadLoca" - "tionRequest\032..flyteidl.service.CreateUpl" - "oadLocationResponse\")\202\323\344\223\002#\"\036/api/v1/dat" - "aproxy/artifact_urn:\001*\022\246\001\n\026CreateDownloa" - "dLocation\022/.flyteidl.service.CreateDownl" - "oadLocationRequest\0320.flyteidl.service.Cr" - "eateDownloadLocationResponse\")\210\002\001\202\323\344\223\002 \022" - "\036/api/v1/dataproxy/artifact_urn\022\233\001\n\022Crea" - "teDownloadLink\022+.flyteidl.service.Create" - "DownloadLinkRequest\032,.flyteidl.service.C" - "reateDownloadLinkResponse\"*\202\323\344\223\002$\"\037/api/" - "v1/dataproxy/artifact_link:\001*\022d\n\007GetData" - "\022 .flyteidl.service.GetDataRequest\032!.fly" - "teidl.service.GetDataResponse\"\024\202\323\344\223\002\016\022\014/" - "api/v1/dataB9Z7github.com/flyteorg/flyte" - "idl/gen/pb-go/flyteidl/serviceb\006proto3" + "\022\025\n\rfilename_root\030\006 \001(\t\"f\n\035CreateDownloa" + "dLocationRequest\022\022\n\nnative_url\030\001 \001(\t\022-\n\n" + "expires_in\030\002 \001(\0132\031.google.protobuf.Durat" + "ion:\002\030\001\"h\n\036CreateDownloadLocationRespons" + "e\022\022\n\nsigned_url\030\001 \001(\t\022.\n\nexpires_at\030\002 \001(" + "\0132\032.google.protobuf.Timestamp:\002\030\001\"\320\001\n\031Cr" + "eateDownloadLinkRequest\0225\n\rartifact_type" + "\030\001 \001(\0162\036.flyteidl.service.ArtifactType\022-" + "\n\nexpires_in\030\002 \001(\0132\031.google.protobuf.Dur" + "ation\022C\n\021node_execution_id\030\003 \001(\0132&.flyte" + "idl.core.NodeExecutionIdentifierH\000B\010\n\006so" + "urce\"\242\001\n\032CreateDownloadLinkResponse\022\026\n\ns" + "igned_url\030\001 \003(\tB\002\030\001\0222\n\nexpires_at\030\002 \001(\0132" + "\032.google.protobuf.TimestampB\002\030\001\0228\n\017pre_s" + "igned_urls\030\003 \001(\0132\037.flyteidl.service.PreS" + "ignedURLs\"S\n\rPreSignedURLs\022\022\n\nsigned_url" + "\030\001 \003(\t\022.\n\nexpires_at\030\002 \001(\0132\032.google.prot" + "obuf.Timestamp\"#\n\016GetDataRequest\022\021\n\tflyt" + "e_url\030\001 \001(\t\"\262\001\n\017GetDataResponse\0220\n\013liter" + "al_map\030\001 \001(\0132\031.flyteidl.core.LiteralMapH" + "\000\022:\n\017pre_signed_urls\030\002 \001(\0132\037.flyteidl.se" + "rvice.PreSignedURLsH\000\022)\n\007literal\030\003 \001(\0132\026" + ".flyteidl.core.LiteralH\000B\006\n\004data*C\n\014Arti" + "factType\022\033\n\027ARTIFACT_TYPE_UNDEFINED\020\000\022\026\n" + "\022ARTIFACT_TYPE_DECK\020\0012\342\004\n\020DataProxyServi" + "ce\022\240\001\n\024CreateUploadLocation\022-.flyteidl.s" + "ervice.CreateUploadLocationRequest\032..fly" + "teidl.service.CreateUploadLocationRespon" + "se\")\202\323\344\223\002#\"\036/api/v1/dataproxy/artifact_u" + "rn:\001*\022\246\001\n\026CreateDownloadLocation\022/.flyte" + "idl.service.CreateDownloadLocationReques" + "t\0320.flyteidl.service.CreateDownloadLocat" + "ionResponse\")\210\002\001\202\323\344\223\002 \022\036/api/v1/dataprox" + "y/artifact_urn\022\233\001\n\022CreateDownloadLink\022+." + "flyteidl.service.CreateDownloadLinkReque" + "st\032,.flyteidl.service.CreateDownloadLink" + "Response\"*\202\323\344\223\002$\"\037/api/v1/dataproxy/arti" + "fact_link:\001*\022d\n\007GetData\022 .flyteidl.servi" + "ce.GetDataRequest\032!.flyteidl.service.Get" + "DataResponse\"\024\202\323\344\223\002\016\022\014/api/v1/dataB9Z7gi" + "thub.com/flyteorg/flyteidl/gen/pb-go/fly" + "teidl/serviceb\006proto3" ; ::google::protobuf::internal::DescriptorTable descriptor_table_flyteidl_2fservice_2fdataproxy_2eproto = { false, InitDefaults_flyteidl_2fservice_2fdataproxy_2eproto, descriptor_table_protodef_flyteidl_2fservice_2fdataproxy_2eproto, - "flyteidl/service/dataproxy.proto", &assign_descriptors_table_flyteidl_2fservice_2fdataproxy_2eproto, 2118, + "flyteidl/service/dataproxy.proto", &assign_descriptors_table_flyteidl_2fservice_2fdataproxy_2eproto, 2141, }; void AddDescriptors_flyteidl_2fservice_2fdataproxy_2eproto() { @@ -883,6 +885,7 @@ const int CreateUploadLocationRequest::kDomainFieldNumber; const int CreateUploadLocationRequest::kFilenameFieldNumber; const int CreateUploadLocationRequest::kExpiresInFieldNumber; const int CreateUploadLocationRequest::kContentMd5FieldNumber; +const int CreateUploadLocationRequest::kFilenameRootFieldNumber; #endif // !defined(_MSC_VER) || _MSC_VER >= 1900 CreateUploadLocationRequest::CreateUploadLocationRequest() @@ -910,6 +913,10 @@ CreateUploadLocationRequest::CreateUploadLocationRequest(const CreateUploadLocat if (from.content_md5().size() > 0) { content_md5_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.content_md5_); } + filename_root_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (from.filename_root().size() > 0) { + filename_root_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.filename_root_); + } if (from.has_expires_in()) { expires_in_ = new ::google::protobuf::Duration(*from.expires_in_); } else { @@ -925,6 +932,7 @@ void CreateUploadLocationRequest::SharedCtor() { domain_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); content_md5_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + filename_root_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); expires_in_ = nullptr; } @@ -938,6 +946,7 @@ void CreateUploadLocationRequest::SharedDtor() { domain_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); filename_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); content_md5_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + filename_root_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); if (this != internal_default_instance()) delete expires_in_; } @@ -960,6 +969,7 @@ void CreateUploadLocationRequest::Clear() { domain_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); filename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); content_md5_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + filename_root_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && expires_in_ != nullptr) { delete expires_in_; } @@ -1056,6 +1066,22 @@ const char* CreateUploadLocationRequest::_InternalParse(const char* begin, const ptr += size; break; } + // string filename_root = 6; + case 6: { + if (static_cast<::google::protobuf::uint8>(tag) != 50) goto handle_unusual; + ptr = ::google::protobuf::io::ReadSize(ptr, &size); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + ctx->extra_parse_data().SetFieldName("flyteidl.service.CreateUploadLocationRequest.filename_root"); + object = msg->mutable_filename_root(); + if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) { + parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8; + goto string_till_end; + } + GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx)); + ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx); + ptr += size; + break; + } default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -1157,6 +1183,21 @@ bool CreateUploadLocationRequest::MergePartialFromCodedStream( break; } + // string filename_root = 6; + case 6: { + if (static_cast< ::google::protobuf::uint8>(tag) == (50 & 0xFF)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_filename_root())); + DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->filename_root().data(), static_cast(this->filename_root().length()), + ::google::protobuf::internal::WireFormatLite::PARSE, + "flyteidl.service.CreateUploadLocationRequest.filename_root")); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -1226,6 +1267,16 @@ void CreateUploadLocationRequest::SerializeWithCachedSizes( 5, this->content_md5(), output); } + // string filename_root = 6; + if (this->filename_root().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->filename_root().data(), static_cast(this->filename_root().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "flyteidl.service.CreateUploadLocationRequest.filename_root"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 6, this->filename_root(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::google::protobuf::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -1286,6 +1337,17 @@ ::google::protobuf::uint8* CreateUploadLocationRequest::InternalSerializeWithCac 5, this->content_md5(), target); } + // string filename_root = 6; + if (this->filename_root().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->filename_root().data(), static_cast(this->filename_root().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "flyteidl.service.CreateUploadLocationRequest.filename_root"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 6, this->filename_root(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -1335,6 +1397,13 @@ size_t CreateUploadLocationRequest::ByteSizeLong() const { this->content_md5()); } + // string filename_root = 6; + if (this->filename_root().size() > 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->filename_root()); + } + // .google.protobuf.Duration expires_in = 4; if (this->has_expires_in()) { total_size += 1 + @@ -1385,6 +1454,10 @@ void CreateUploadLocationRequest::MergeFrom(const CreateUploadLocationRequest& f content_md5_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.content_md5_); } + if (from.filename_root().size() > 0) { + + filename_root_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.filename_root_); + } if (from.has_expires_in()) { mutable_expires_in()->::google::protobuf::Duration::MergeFrom(from.expires_in()); } @@ -1423,6 +1496,8 @@ void CreateUploadLocationRequest::InternalSwap(CreateUploadLocationRequest* othe GetArenaNoVirtual()); content_md5_.Swap(&other->content_md5_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); + filename_root_.Swap(&other->filename_root_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); swap(expires_in_, other->expires_in_); } diff --git a/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.h b/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.h index cd5f8cfa3..3332d747b 100644 --- a/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.h +++ b/flyteidl/gen/pb-cpp/flyteidl/service/dataproxy.pb.h @@ -420,6 +420,20 @@ class CreateUploadLocationRequest final : ::std::string* release_content_md5(); void set_allocated_content_md5(::std::string* content_md5); + // string filename_root = 6; + void clear_filename_root(); + static const int kFilenameRootFieldNumber = 6; + const ::std::string& filename_root() const; + void set_filename_root(const ::std::string& value); + #if LANG_CXX11 + void set_filename_root(::std::string&& value); + #endif + void set_filename_root(const char* value); + void set_filename_root(const char* value, size_t size); + ::std::string* mutable_filename_root(); + ::std::string* release_filename_root(); + void set_allocated_filename_root(::std::string* filename_root); + // .google.protobuf.Duration expires_in = 4; bool has_expires_in() const; void clear_expires_in(); @@ -438,6 +452,7 @@ class CreateUploadLocationRequest final : ::google::protobuf::internal::ArenaStringPtr domain_; ::google::protobuf::internal::ArenaStringPtr filename_; ::google::protobuf::internal::ArenaStringPtr content_md5_; + ::google::protobuf::internal::ArenaStringPtr filename_root_; ::google::protobuf::Duration* expires_in_; mutable ::google::protobuf::internal::CachedSize _cached_size_; friend struct ::TableStruct_flyteidl_2fservice_2fdataproxy_2eproto; @@ -1836,6 +1851,59 @@ inline void CreateUploadLocationRequest::set_allocated_content_md5(::std::string // @@protoc_insertion_point(field_set_allocated:flyteidl.service.CreateUploadLocationRequest.content_md5) } +// string filename_root = 6; +inline void CreateUploadLocationRequest::clear_filename_root() { + filename_root_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline const ::std::string& CreateUploadLocationRequest::filename_root() const { + // @@protoc_insertion_point(field_get:flyteidl.service.CreateUploadLocationRequest.filename_root) + return filename_root_.GetNoArena(); +} +inline void CreateUploadLocationRequest::set_filename_root(const ::std::string& value) { + + filename_root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:flyteidl.service.CreateUploadLocationRequest.filename_root) +} +#if LANG_CXX11 +inline void CreateUploadLocationRequest::set_filename_root(::std::string&& value) { + + filename_root_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:flyteidl.service.CreateUploadLocationRequest.filename_root) +} +#endif +inline void CreateUploadLocationRequest::set_filename_root(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + filename_root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:flyteidl.service.CreateUploadLocationRequest.filename_root) +} +inline void CreateUploadLocationRequest::set_filename_root(const char* value, size_t size) { + + filename_root_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:flyteidl.service.CreateUploadLocationRequest.filename_root) +} +inline ::std::string* CreateUploadLocationRequest::mutable_filename_root() { + + // @@protoc_insertion_point(field_mutable:flyteidl.service.CreateUploadLocationRequest.filename_root) + return filename_root_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* CreateUploadLocationRequest::release_filename_root() { + // @@protoc_insertion_point(field_release:flyteidl.service.CreateUploadLocationRequest.filename_root) + + return filename_root_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void CreateUploadLocationRequest::set_allocated_filename_root(::std::string* filename_root) { + if (filename_root != nullptr) { + + } else { + + } + filename_root_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), filename_root); + // @@protoc_insertion_point(field_set_allocated:flyteidl.service.CreateUploadLocationRequest.filename_root) +} + // ------------------------------------------------------------------- // CreateDownloadLocationRequest diff --git a/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go b/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go index 6497aa469..c59c3099f 100644 --- a/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go @@ -116,6 +116,10 @@ func (m *CreateUploadLocationResponse) GetExpiresAt() *timestamp.Timestamp { } // CreateUploadLocationRequest specified request for the CreateUploadLocation API. +// The implementation in data proxy service will create the s3 location with some server side configured prefixes, +// and then: +// - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR +// - project/domain/filename_root (if present)/filename (if present). type CreateUploadLocationRequest struct { // Project to create the upload location for // +required @@ -133,7 +137,12 @@ type CreateUploadLocationRequest struct { // ContentMD5 restricts the upload location to the specific MD5 provided. The ContentMD5 will also appear in the // generated path. // +required - ContentMd5 []byte `protobuf:"bytes,5,opt,name=content_md5,json=contentMd5,proto3" json:"content_md5,omitempty"` + ContentMd5 []byte `protobuf:"bytes,5,opt,name=content_md5,json=contentMd5,proto3" json:"content_md5,omitempty"` + // If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included + // this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix + // in data proxy config. This option is useful when uploading multiple files. + // +optional + FilenameRoot string `protobuf:"bytes,6,opt,name=filename_root,json=filenameRoot,proto3" json:"filename_root,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -199,6 +208,13 @@ func (m *CreateUploadLocationRequest) GetContentMd5() []byte { return nil } +func (m *CreateUploadLocationRequest) GetFilenameRoot() string { + if m != nil { + return m.FilenameRoot + } + return "" +} + // CreateDownloadLocationRequest specified request for the CreateDownloadLocation API. // // Deprecated: Do not use. @@ -656,63 +672,64 @@ func init() { func init() { proto.RegisterFile("flyteidl/service/dataproxy.proto", fileDescriptor_bffb71366d75dab0) } var fileDescriptor_bffb71366d75dab0 = []byte{ - // 887 bytes of a gzipped FileDescriptorProto + // 907 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x6f, 0xe3, 0x44, - 0x14, 0xcf, 0xa4, 0xa5, 0x6d, 0x5e, 0xff, 0x32, 0x5a, 0x95, 0xac, 0xdb, 0x6d, 0xb3, 0x06, 0xa1, - 0x52, 0xa8, 0x0d, 0x45, 0x2b, 0xd8, 0x15, 0x1c, 0xda, 0x24, 0xbb, 0x8d, 0xe8, 0x56, 0x95, 0x37, - 0x3d, 0xc0, 0xc5, 0x9a, 0xda, 0x93, 0x30, 0xac, 0x33, 0x63, 0xc6, 0x93, 0xd2, 0x48, 0x88, 0x03, - 0x27, 0xee, 0x1c, 0x38, 0x70, 0xe1, 0xc4, 0x47, 0x81, 0x0f, 0x80, 0xc4, 0x07, 0x40, 0x7c, 0x0a, - 0x4e, 0xc8, 0xf6, 0xd8, 0x89, 0x93, 0xb4, 0x9b, 0x22, 0x8e, 0xcf, 0xef, 0x67, 0xff, 0x7e, 0xef, - 0xf7, 0xde, 0xbc, 0x31, 0xd4, 0x3a, 0xc1, 0x40, 0x51, 0xe6, 0x07, 0x76, 0x44, 0xe5, 0x15, 0xf3, - 0xa8, 0xed, 0x13, 0x45, 0x42, 0x29, 0xae, 0x07, 0x56, 0x28, 0x85, 0x12, 0x78, 0x23, 0x43, 0x58, - 0x1a, 0x61, 0x6c, 0x77, 0x85, 0xe8, 0x06, 0xd4, 0x26, 0x21, 0xb3, 0x09, 0xe7, 0x42, 0x11, 0xc5, - 0x04, 0x8f, 0x52, 0xbc, 0xb1, 0xa3, 0xb3, 0x49, 0x74, 0xd9, 0xef, 0xd8, 0x7e, 0x5f, 0x26, 0x00, - 0x9d, 0xdf, 0x1d, 0xcf, 0x2b, 0xd6, 0xa3, 0x91, 0x22, 0xbd, 0x30, 0xfb, 0x40, 0x2e, 0xc9, 0x13, - 0x92, 0xda, 0xcc, 0xa7, 0x5c, 0xb1, 0x0e, 0xa3, 0x52, 0xe7, 0xb7, 0x8b, 0xf9, 0x80, 0x29, 0x2a, - 0x49, 0xa0, 0xe9, 0xcd, 0x9f, 0x10, 0x6c, 0xd7, 0x25, 0x25, 0x8a, 0x5e, 0x84, 0x81, 0x20, 0xfe, - 0xa9, 0xf0, 0x12, 0x76, 0x87, 0x46, 0xa1, 0xe0, 0x11, 0xc5, 0x0f, 0x00, 0x22, 0xd6, 0xe5, 0xd4, - 0x77, 0xfb, 0x32, 0xa8, 0xa2, 0x1a, 0xda, 0xab, 0x38, 0x95, 0xf4, 0xc9, 0x85, 0x0c, 0xe2, 0x34, - 0x27, 0x8a, 0x5d, 0xd1, 0x24, 0x5d, 0x4e, 0xd3, 0xe9, 0x93, 0x38, 0xfd, 0x18, 0x80, 0x5e, 0x87, - 0x4c, 0xd2, 0xc8, 0x25, 0xaa, 0x3a, 0x57, 0x43, 0x7b, 0xcb, 0x87, 0x86, 0x95, 0x96, 0x64, 0x65, - 0x25, 0x59, 0xed, 0xac, 0x24, 0xa7, 0xa2, 0xd1, 0x47, 0xca, 0xfc, 0x0d, 0xc1, 0xd6, 0x74, 0x65, - 0x5f, 0xf7, 0x69, 0xa4, 0x70, 0x15, 0x16, 0x43, 0x29, 0xbe, 0xa2, 0x9e, 0xd2, 0xaa, 0xb2, 0x10, - 0x6f, 0xc2, 0x82, 0x2f, 0x7a, 0x84, 0x71, 0xad, 0x47, 0x47, 0xd8, 0x80, 0xa5, 0x0e, 0x0b, 0x28, - 0x27, 0x3d, 0x9a, 0x48, 0xa9, 0x38, 0x79, 0x8c, 0x3f, 0x1e, 0x0a, 0x65, 0xbc, 0x3a, 0x9f, 0x08, - 0xbd, 0x3f, 0x21, 0xb4, 0xa1, 0x7b, 0x93, 0xeb, 0x6c, 0x71, 0xbc, 0x0b, 0xcb, 0x9e, 0xe0, 0x8a, - 0x72, 0xe5, 0xf6, 0xfc, 0x47, 0xd5, 0xd7, 0x6a, 0x68, 0x6f, 0xc5, 0x01, 0xfd, 0xe8, 0xb9, 0xff, - 0xc8, 0xfc, 0x16, 0x1e, 0xa4, 0x75, 0x34, 0xc4, 0x37, 0x7c, 0x5a, 0x25, 0x45, 0x0f, 0xd1, 0xb8, - 0x87, 0x45, 0x69, 0xe5, 0xd9, 0xa5, 0x3d, 0x29, 0x57, 0x91, 0xf9, 0x1d, 0xec, 0xdc, 0xc4, 0x3e, - 0x5b, 0x87, 0x8b, 0x2d, 0x2c, 0xdf, 0xa1, 0x85, 0x09, 0xff, 0x3f, 0x08, 0xee, 0x8f, 0x09, 0x60, - 0xfc, 0x65, 0x56, 0x7a, 0x1d, 0x56, 0x89, 0x54, 0xac, 0x43, 0x3c, 0xe5, 0xaa, 0x41, 0x48, 0x13, - 0xfa, 0xb5, 0xc3, 0x1d, 0x6b, 0xfc, 0x14, 0x59, 0x47, 0x1a, 0xd6, 0x1e, 0x84, 0xd4, 0x59, 0x21, - 0x23, 0xd1, 0x7f, 0x37, 0x08, 0xb7, 0xe1, 0x75, 0x2e, 0x7c, 0xea, 0xd2, 0x6b, 0xea, 0xf5, 0xe3, - 0xa4, 0xcb, 0x7c, 0x3d, 0xa5, 0x6f, 0x0f, 0x25, 0xc4, 0xe7, 0xc6, 0x3a, 0x13, 0x3e, 0x6d, 0x66, - 0xb0, 0x56, 0x7e, 0xc8, 0x4e, 0x4a, 0xce, 0x3a, 0x2f, 0xa6, 0x8e, 0x97, 0x60, 0x21, 0x12, 0x7d, - 0xe9, 0x51, 0xf3, 0x77, 0x04, 0xc6, 0xb4, 0xe2, 0xb5, 0xf3, 0x0f, 0xc7, 0x9c, 0x9f, 0xdb, 0xab, - 0x1c, 0x97, 0xab, 0x68, 0xd4, 0xfd, 0x4f, 0xef, 0xe6, 0x7e, 0xfa, 0x7a, 0xde, 0x01, 0xfc, 0x0c, - 0xd6, 0x43, 0x49, 0xdd, 0x21, 0x4b, 0xa4, 0xcb, 0xdb, 0x9d, 0x74, 0xf8, 0x5c, 0xd2, 0x17, 0x29, - 0xaf, 0x73, 0x1a, 0x39, 0xab, 0x61, 0x1e, 0xca, 0x20, 0x32, 0x19, 0xac, 0x16, 0xf2, 0x13, 0x53, - 0x33, 0xf7, 0x7f, 0x4d, 0x8d, 0x79, 0x00, 0x6b, 0xcf, 0xa8, 0x6a, 0x10, 0x45, 0xb2, 0x29, 0xd9, - 0x82, 0x4a, 0xa2, 0x76, 0x64, 0x40, 0x97, 0x92, 0x07, 0x17, 0x32, 0x30, 0xff, 0x44, 0xb0, 0x9e, - 0xe3, 0xb5, 0xb1, 0x9f, 0xc0, 0xb2, 0xde, 0x73, 0x6e, 0x8f, 0x84, 0xc9, 0x2b, 0xf1, 0x48, 0x14, - 0x3b, 0x7a, 0x9a, 0x22, 0x9e, 0x93, 0xf0, 0xa4, 0xe4, 0x40, 0x90, 0x47, 0xb8, 0x35, 0x69, 0x5a, - 0x79, 0x26, 0xd3, 0x4e, 0x4a, 0x63, 0xb6, 0xe1, 0x43, 0x58, 0xd4, 0x1f, 0xd6, 0xbe, 0x6f, 0x4e, - 0x17, 0x71, 0x52, 0x72, 0x32, 0xe0, 0xf1, 0x02, 0xcc, 0xc7, 0x97, 0xca, 0x7e, 0x1d, 0x56, 0x46, - 0x87, 0x1e, 0x6f, 0xc1, 0x1b, 0x47, 0x4e, 0xbb, 0xf5, 0xf4, 0xa8, 0xde, 0x76, 0xdb, 0x9f, 0x9f, - 0x37, 0xdd, 0x8b, 0xb3, 0x46, 0xf3, 0x69, 0xeb, 0xac, 0xd9, 0xd8, 0x28, 0xe1, 0x4d, 0xc0, 0xc5, - 0x64, 0xa3, 0x59, 0xff, 0x6c, 0x03, 0x1d, 0xfe, 0x35, 0x0f, 0x1b, 0xb1, 0x35, 0xe7, 0xf1, 0x15, - 0xf5, 0x22, 0x15, 0x8d, 0x7f, 0x41, 0x70, 0x6f, 0xda, 0x6a, 0xc5, 0x07, 0x93, 0x05, 0xde, 0xb2, - 0x82, 0x0d, 0x6b, 0x56, 0x78, 0xda, 0x16, 0xf3, 0x9d, 0xef, 0xff, 0xf8, 0xfb, 0xc7, 0xf2, 0x9b, - 0xe6, 0x4e, 0x72, 0x17, 0x5e, 0x7d, 0x30, 0xbc, 0x3c, 0xed, 0x7c, 0x0b, 0xf4, 0x25, 0x7f, 0x82, - 0xf6, 0xf1, 0xaf, 0x08, 0x36, 0xa7, 0xef, 0x2d, 0x6c, 0xdf, 0xc4, 0x7a, 0xc3, 0x7e, 0x35, 0xde, - 0x9f, 0xfd, 0x85, 0x82, 0xd0, 0x1a, 0x7e, 0x85, 0xd0, 0x1f, 0xca, 0x08, 0xff, 0x8c, 0x00, 0x4f, - 0x1e, 0x71, 0xfc, 0xee, 0x2b, 0x39, 0x87, 0x5b, 0xd0, 0x78, 0x6f, 0x36, 0xb0, 0x16, 0xb7, 0x9f, - 0x88, 0x7b, 0xcb, 0xdc, 0xbd, 0x45, 0x5c, 0xc0, 0xf8, 0xcb, 0xd8, 0x46, 0x1f, 0x16, 0xf5, 0xd9, - 0xc0, 0xb5, 0x49, 0x92, 0xe2, 0x31, 0x33, 0x1e, 0xde, 0x82, 0xd0, 0xdc, 0xf7, 0x12, 0xee, 0x35, - 0xbc, 0x32, 0xca, 0x7d, 0xfc, 0xf8, 0x8b, 0x8f, 0xba, 0x4c, 0x7d, 0xd9, 0xbf, 0xb4, 0x3c, 0xd1, - 0xb3, 0x93, 0x8f, 0x08, 0xd9, 0xb5, 0xf3, 0x1f, 0x8f, 0x2e, 0xe5, 0x76, 0x78, 0x79, 0xd0, 0x15, - 0xf6, 0xf8, 0xef, 0xd3, 0xe5, 0x42, 0xb2, 0x0b, 0x3e, 0xfc, 0x37, 0x00, 0x00, 0xff, 0xff, 0x0e, - 0x26, 0x30, 0xb8, 0x59, 0x09, 0x00, 0x00, + 0x14, 0xef, 0xa4, 0x25, 0x6d, 0x5e, 0xd3, 0x3f, 0x8c, 0x56, 0x25, 0xeb, 0x76, 0xdb, 0xac, 0x17, + 0xa1, 0x52, 0xa8, 0x0d, 0x45, 0x2b, 0xd8, 0x15, 0x1c, 0xda, 0x24, 0xbb, 0x8d, 0xe8, 0x56, 0x95, + 0x37, 0x3d, 0xc0, 0xc5, 0x9a, 0xda, 0x93, 0x30, 0xac, 0x33, 0x63, 0xc6, 0x93, 0xd2, 0x4a, 0x88, + 0x03, 0x27, 0xee, 0x1c, 0x38, 0x70, 0xe1, 0xc4, 0x57, 0xe1, 0x03, 0x20, 0xf1, 0x01, 0x10, 0x37, + 0xbe, 0x01, 0x27, 0x64, 0x7b, 0xec, 0xc4, 0x49, 0xda, 0xcd, 0xa2, 0x3d, 0xbe, 0x79, 0x3f, 0xcf, + 0xef, 0xf7, 0x7e, 0xf3, 0xe6, 0x8d, 0xa1, 0xde, 0x0d, 0xae, 0x15, 0x65, 0x7e, 0x60, 0x47, 0x54, + 0x5e, 0x32, 0x8f, 0xda, 0x3e, 0x51, 0x24, 0x94, 0xe2, 0xea, 0xda, 0x0a, 0xa5, 0x50, 0x02, 0xaf, + 0x67, 0x08, 0x4b, 0x23, 0x8c, 0xad, 0x9e, 0x10, 0xbd, 0x80, 0xda, 0x24, 0x64, 0x36, 0xe1, 0x5c, + 0x28, 0xa2, 0x98, 0xe0, 0x51, 0x8a, 0x37, 0xb6, 0x75, 0x36, 0x89, 0x2e, 0x06, 0x5d, 0xdb, 0x1f, + 0xc8, 0x04, 0xa0, 0xf3, 0x3b, 0xe3, 0x79, 0xc5, 0xfa, 0x34, 0x52, 0xa4, 0x1f, 0x66, 0x1b, 0xe4, + 0x92, 0x3c, 0x21, 0xa9, 0xcd, 0x7c, 0xca, 0x15, 0xeb, 0x32, 0x2a, 0x75, 0x7e, 0xab, 0x98, 0x0f, + 0x98, 0xa2, 0x92, 0x04, 0x9a, 0xde, 0xfc, 0x19, 0xc1, 0x56, 0x43, 0x52, 0xa2, 0xe8, 0x79, 0x18, + 0x08, 0xe2, 0x9f, 0x08, 0x2f, 0x61, 0x77, 0x68, 0x14, 0x0a, 0x1e, 0x51, 0x7c, 0x0f, 0x20, 0x62, + 0x3d, 0x4e, 0x7d, 0x77, 0x20, 0x83, 0x1a, 0xaa, 0xa3, 0xdd, 0x8a, 0x53, 0x49, 0x57, 0xce, 0x65, + 0x10, 0xa7, 0x39, 0x51, 0xec, 0x92, 0x26, 0xe9, 0x52, 0x9a, 0x4e, 0x57, 0xe2, 0xf4, 0x23, 0x00, + 0x7a, 0x15, 0x32, 0x49, 0x23, 0x97, 0xa8, 0xda, 0x7c, 0x1d, 0xed, 0x2e, 0x1f, 0x18, 0x56, 0x5a, + 0x92, 0x95, 0x95, 0x64, 0x75, 0xb2, 0x92, 0x9c, 0x8a, 0x46, 0x1f, 0x2a, 0xf3, 0x1f, 0x04, 0x9b, + 0xd3, 0x95, 0x7d, 0x33, 0xa0, 0x91, 0xc2, 0x35, 0x58, 0x0c, 0xa5, 0xf8, 0x9a, 0x7a, 0x4a, 0xab, + 0xca, 0x42, 0xbc, 0x01, 0x65, 0x5f, 0xf4, 0x09, 0xe3, 0x5a, 0x8f, 0x8e, 0xb0, 0x01, 0x4b, 0x5d, + 0x16, 0x50, 0x4e, 0xfa, 0x34, 0x91, 0x52, 0x71, 0xf2, 0x18, 0x7f, 0x32, 0x14, 0xca, 0x78, 0x6d, + 0x21, 0x11, 0x7a, 0x77, 0x42, 0x68, 0x53, 0x9f, 0x4d, 0xae, 0xb3, 0xcd, 0xf1, 0x0e, 0x2c, 0x7b, + 0x82, 0x2b, 0xca, 0x95, 0xdb, 0xf7, 0x1f, 0xd6, 0xde, 0xa8, 0xa3, 0xdd, 0xaa, 0x03, 0x7a, 0xe9, + 0x99, 0xff, 0x10, 0x3f, 0x80, 0x95, 0x8c, 0xc6, 0x95, 0x42, 0xa8, 0x5a, 0x39, 0xe1, 0xae, 0x66, + 0x8b, 0x8e, 0x10, 0xca, 0xfc, 0x0e, 0xee, 0xa5, 0xc5, 0x36, 0xc5, 0xb7, 0x7c, 0x5a, 0xb9, 0x45, + 0xa3, 0xd1, 0xb8, 0xd1, 0x45, 0xfd, 0xa5, 0xd9, 0xf5, 0x3f, 0x2e, 0xd5, 0x90, 0xf9, 0x3d, 0x6c, + 0xdf, 0xc4, 0x3e, 0x5b, 0x1b, 0x14, 0xcf, 0xb9, 0xf4, 0x0a, 0xe7, 0x9c, 0xf0, 0xff, 0x8b, 0xe0, + 0xee, 0x98, 0x00, 0xc6, 0x5f, 0x64, 0xa5, 0x37, 0x60, 0x85, 0x48, 0xc5, 0xba, 0xc4, 0x53, 0xae, + 0xba, 0x0e, 0x69, 0x42, 0xbf, 0x7a, 0xb0, 0x6d, 0x8d, 0x5f, 0x35, 0xeb, 0x50, 0xc3, 0x3a, 0xd7, + 0x21, 0x75, 0xaa, 0x64, 0x24, 0xfa, 0xff, 0x06, 0xe1, 0x0e, 0xbc, 0xc9, 0x85, 0x4f, 0x5d, 0x7a, + 0x45, 0xbd, 0x41, 0x9c, 0x74, 0x99, 0xaf, 0x5b, 0xf9, 0x9d, 0xa1, 0x84, 0xf8, 0x72, 0x59, 0xa7, + 0xc2, 0xa7, 0xad, 0x0c, 0xd6, 0xce, 0x6f, 0xe2, 0xf1, 0x9c, 0xb3, 0xc6, 0x8b, 0xa9, 0xa3, 0x25, + 0x28, 0x47, 0x62, 0x20, 0x3d, 0x6a, 0xfe, 0x8e, 0xc0, 0x98, 0x56, 0xbc, 0x76, 0xfe, 0xfe, 0x98, + 0xf3, 0xf3, 0xbb, 0x95, 0xa3, 0x52, 0x0d, 0x8d, 0xba, 0xff, 0xd9, 0xab, 0xb9, 0x9f, 0x7e, 0x9e, + 0x9f, 0x00, 0x7e, 0x0a, 0x6b, 0xa1, 0xa4, 0xee, 0x90, 0x25, 0xd2, 0xe5, 0xed, 0x4c, 0x3a, 0x7c, + 0x26, 0xe9, 0xf3, 0x94, 0xd7, 0x39, 0x89, 0x9c, 0x95, 0x30, 0x0f, 0x65, 0x10, 0x99, 0x0c, 0x56, + 0x0a, 0xf9, 0x89, 0xae, 0x99, 0x7f, 0x5d, 0x5d, 0x63, 0xee, 0xc3, 0xea, 0x53, 0xaa, 0x9a, 0x44, + 0x91, 0xac, 0x4b, 0x36, 0xa1, 0x92, 0xa8, 0x1d, 0x69, 0xd0, 0xa5, 0x64, 0xe1, 0x5c, 0x06, 0xe6, + 0x9f, 0x08, 0xd6, 0x72, 0xbc, 0x36, 0xf6, 0x53, 0x58, 0xd6, 0xc3, 0xd0, 0xed, 0x93, 0x30, 0xf9, + 0x24, 0x6e, 0x89, 0xe2, 0x89, 0x9e, 0xa4, 0x88, 0x67, 0x24, 0x3c, 0x9e, 0x73, 0x20, 0xc8, 0x23, + 0xdc, 0x9e, 0x34, 0xad, 0x34, 0x93, 0x69, 0xc7, 0x73, 0x63, 0xb6, 0xe1, 0x03, 0x58, 0xd4, 0x1b, + 0x6b, 0xdf, 0x37, 0xa6, 0x8b, 0x38, 0x9e, 0x73, 0x32, 0xe0, 0x51, 0x19, 0x16, 0xe2, 0x97, 0x67, + 0xaf, 0x01, 0xd5, 0xd1, 0xa6, 0xc7, 0x9b, 0xf0, 0xd6, 0xa1, 0xd3, 0x69, 0x3f, 0x39, 0x6c, 0x74, + 0xdc, 0xce, 0x17, 0x67, 0x2d, 0xf7, 0xfc, 0xb4, 0xd9, 0x7a, 0xd2, 0x3e, 0x6d, 0x35, 0xd7, 0xe7, + 0xf0, 0x06, 0xe0, 0x62, 0xb2, 0xd9, 0x6a, 0x7c, 0xbe, 0x8e, 0x0e, 0xfe, 0x5a, 0x80, 0xf5, 0xd8, + 0x9a, 0xb3, 0xf8, 0x1d, 0x7b, 0x9e, 0x8a, 0xc6, 0xbf, 0x22, 0xb8, 0x33, 0x6d, 0xfe, 0xe2, 0xfd, + 0xc9, 0x02, 0x6f, 0x99, 0xd3, 0x86, 0x35, 0x2b, 0x3c, 0x3d, 0x16, 0xf3, 0xdd, 0x1f, 0xfe, 0xf8, + 0xfb, 0xa7, 0xd2, 0x03, 0x73, 0x3b, 0x79, 0x30, 0x2f, 0x3f, 0x1c, 0xbe, 0xb0, 0x76, 0x3e, 0x05, + 0x06, 0x92, 0x3f, 0x46, 0x7b, 0xf8, 0x37, 0x04, 0x1b, 0xd3, 0xe7, 0x16, 0xb6, 0x6f, 0x62, 0xbd, + 0x61, 0xbe, 0x1a, 0x1f, 0xcc, 0xfe, 0x41, 0x41, 0x68, 0x1d, 0xbf, 0x44, 0xe8, 0x8f, 0x25, 0x84, + 0x7f, 0x41, 0x80, 0x27, 0xaf, 0x38, 0x7e, 0xef, 0xa5, 0x9c, 0xc3, 0x29, 0x68, 0xbc, 0x3f, 0x1b, + 0x58, 0x8b, 0xdb, 0x4b, 0xc4, 0xbd, 0x6d, 0xee, 0xdc, 0x22, 0x2e, 0x60, 0xfc, 0x45, 0x6c, 0xa3, + 0x0f, 0x8b, 0xfa, 0x6e, 0xe0, 0xfa, 0x24, 0x49, 0xf1, 0x9a, 0x19, 0xf7, 0x6f, 0x41, 0x68, 0xee, + 0x3b, 0x09, 0xf7, 0x2a, 0xae, 0x8e, 0x72, 0x1f, 0x3d, 0xfa, 0xf2, 0xe3, 0x1e, 0x53, 0x5f, 0x0d, + 0x2e, 0x2c, 0x4f, 0xf4, 0xed, 0x64, 0x13, 0x21, 0x7b, 0x76, 0xfe, 0x77, 0xd2, 0xa3, 0xdc, 0x0e, + 0x2f, 0xf6, 0x7b, 0xc2, 0x1e, 0xff, 0xc7, 0xba, 0x28, 0x27, 0xb3, 0xe0, 0xa3, 0xff, 0x02, 0x00, + 0x00, 0xff, 0xff, 0x31, 0x6c, 0xda, 0xe4, 0x7e, 0x09, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/flyteidl/gen/pb-go/flyteidl/service/dataproxy.swagger.json b/flyteidl/gen/pb-go/flyteidl/service/dataproxy.swagger.json index 3afff352e..973243441 100644 --- a/flyteidl/gen/pb-go/flyteidl/service/dataproxy.swagger.json +++ b/flyteidl/gen/pb-go/flyteidl/service/dataproxy.swagger.json @@ -722,9 +722,13 @@ "type": "string", "format": "byte", "title": "ContentMD5 restricts the upload location to the specific MD5 provided. The ContentMD5 will also appear in the\ngenerated path.\n+required" + }, + "filename_root": { + "type": "string", + "title": "If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included\nthis makes the upload location deterministic. The native url will still be prefixed by the upload location prefix\nin data proxy config. This option is useful when uploading multiple files.\n+optional" } }, - "description": "CreateUploadLocationRequest specified request for the CreateUploadLocation API." + "description": "CreateUploadLocationRequest specified request for the CreateUploadLocation API.\nThe implementation in data proxy service will create the s3 location with some server side configured prefixes,\nand then:\n - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR\n - project/domain/filename_root (if present)/filename (if present)." }, "serviceCreateUploadLocationResponse": { "type": "object", diff --git a/flyteidl/gen/pb-java/flyteidl/service/Dataproxy.java b/flyteidl/gen/pb-java/flyteidl/service/Dataproxy.java index 0b9b5d853..431c1d380 100644 --- a/flyteidl/gen/pb-java/flyteidl/service/Dataproxy.java +++ b/flyteidl/gen/pb-java/flyteidl/service/Dataproxy.java @@ -1251,10 +1251,38 @@ public interface CreateUploadLocationRequestOrBuilder extends * bytes content_md5 = 5; */ com.google.protobuf.ByteString getContentMd5(); + + /** + *
+     * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+     * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+     * in data proxy config. This option is useful when uploading multiple files.
+     * +optional
+     * 
+ * + * string filename_root = 6; + */ + java.lang.String getFilenameRoot(); + /** + *
+     * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+     * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+     * in data proxy config. This option is useful when uploading multiple files.
+     * +optional
+     * 
+ * + * string filename_root = 6; + */ + com.google.protobuf.ByteString + getFilenameRootBytes(); } /** *
    * CreateUploadLocationRequest specified request for the CreateUploadLocation API.
+   * The implementation in data proxy service will create the s3 location with some server side configured prefixes,
+   * and then:
+   *   - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR
+   *   - project/domain/filename_root (if present)/filename (if present).
    * 
* * Protobuf type {@code flyteidl.service.CreateUploadLocationRequest} @@ -1273,6 +1301,7 @@ private CreateUploadLocationRequest() { domain_ = ""; filename_ = ""; contentMd5_ = com.google.protobuf.ByteString.EMPTY; + filenameRoot_ = ""; } @java.lang.Override @@ -1335,6 +1364,12 @@ private CreateUploadLocationRequest( contentMd5_ = input.readBytes(); break; } + case 50: { + java.lang.String s = input.readStringRequireUtf8(); + + filenameRoot_ = s; + break; + } default: { if (!parseUnknownField( input, unknownFields, extensionRegistry, tag)) { @@ -1553,6 +1588,54 @@ public com.google.protobuf.ByteString getContentMd5() { return contentMd5_; } + public static final int FILENAME_ROOT_FIELD_NUMBER = 6; + private volatile java.lang.Object filenameRoot_; + /** + *
+     * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+     * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+     * in data proxy config. This option is useful when uploading multiple files.
+     * +optional
+     * 
+ * + * string filename_root = 6; + */ + public java.lang.String getFilenameRoot() { + java.lang.Object ref = filenameRoot_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + filenameRoot_ = s; + return s; + } + } + /** + *
+     * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+     * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+     * in data proxy config. This option is useful when uploading multiple files.
+     * +optional
+     * 
+ * + * string filename_root = 6; + */ + public com.google.protobuf.ByteString + getFilenameRootBytes() { + java.lang.Object ref = filenameRoot_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + filenameRoot_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + private byte memoizedIsInitialized = -1; @java.lang.Override public final boolean isInitialized() { @@ -1582,6 +1665,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (!contentMd5_.isEmpty()) { output.writeBytes(5, contentMd5_); } + if (!getFilenameRootBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 6, filenameRoot_); + } unknownFields.writeTo(output); } @@ -1608,6 +1694,9 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeBytesSize(5, contentMd5_); } + if (!getFilenameRootBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(6, filenameRoot_); + } size += unknownFields.getSerializedSize(); memoizedSize = size; return size; @@ -1636,6 +1725,8 @@ public boolean equals(final java.lang.Object obj) { } if (!getContentMd5() .equals(other.getContentMd5())) return false; + if (!getFilenameRoot() + .equals(other.getFilenameRoot())) return false; if (!unknownFields.equals(other.unknownFields)) return false; return true; } @@ -1659,6 +1750,8 @@ public int hashCode() { } hash = (37 * hash) + CONTENT_MD5_FIELD_NUMBER; hash = (53 * hash) + getContentMd5().hashCode(); + hash = (37 * hash) + FILENAME_ROOT_FIELD_NUMBER; + hash = (53 * hash) + getFilenameRoot().hashCode(); hash = (29 * hash) + unknownFields.hashCode(); memoizedHashCode = hash; return hash; @@ -1757,6 +1850,10 @@ protected Builder newBuilderForType( /** *
      * CreateUploadLocationRequest specified request for the CreateUploadLocation API.
+     * The implementation in data proxy service will create the s3 location with some server side configured prefixes,
+     * and then:
+     *   - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR
+     *   - project/domain/filename_root (if present)/filename (if present).
      * 
* * Protobuf type {@code flyteidl.service.CreateUploadLocationRequest} @@ -1810,6 +1907,8 @@ public Builder clear() { } contentMd5_ = com.google.protobuf.ByteString.EMPTY; + filenameRoot_ = ""; + return this; } @@ -1845,6 +1944,7 @@ public flyteidl.service.Dataproxy.CreateUploadLocationRequest buildPartial() { result.expiresIn_ = expiresInBuilder_.build(); } result.contentMd5_ = contentMd5_; + result.filenameRoot_ = filenameRoot_; onBuilt(); return result; } @@ -1911,6 +2011,10 @@ public Builder mergeFrom(flyteidl.service.Dataproxy.CreateUploadLocationRequest if (other.getContentMd5() != com.google.protobuf.ByteString.EMPTY) { setContentMd5(other.getContentMd5()); } + if (!other.getFilenameRoot().isEmpty()) { + filenameRoot_ = other.filenameRoot_; + onChanged(); + } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; @@ -2439,6 +2543,110 @@ public Builder clearContentMd5() { onChanged(); return this; } + + private java.lang.Object filenameRoot_ = ""; + /** + *
+       * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+       * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+       * in data proxy config. This option is useful when uploading multiple files.
+       * +optional
+       * 
+ * + * string filename_root = 6; + */ + public java.lang.String getFilenameRoot() { + java.lang.Object ref = filenameRoot_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + filenameRoot_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + *
+       * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+       * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+       * in data proxy config. This option is useful when uploading multiple files.
+       * +optional
+       * 
+ * + * string filename_root = 6; + */ + public com.google.protobuf.ByteString + getFilenameRootBytes() { + java.lang.Object ref = filenameRoot_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + filenameRoot_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + *
+       * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+       * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+       * in data proxy config. This option is useful when uploading multiple files.
+       * +optional
+       * 
+ * + * string filename_root = 6; + */ + public Builder setFilenameRoot( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + + filenameRoot_ = value; + onChanged(); + return this; + } + /** + *
+       * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+       * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+       * in data proxy config. This option is useful when uploading multiple files.
+       * +optional
+       * 
+ * + * string filename_root = 6; + */ + public Builder clearFilenameRoot() { + + filenameRoot_ = getDefaultInstance().getFilenameRoot(); + onChanged(); + return this; + } + /** + *
+       * If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included
+       * this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix
+       * in data proxy config. This option is useful when uploading multiple files.
+       * +optional
+       * 
+ * + * string filename_root = 6; + */ + public Builder setFilenameRootBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + filenameRoot_ = value; + onChanged(); + return this; + } @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { @@ -9545,51 +9753,52 @@ public flyteidl.service.Dataproxy.GetDataResponse getDefaultInstanceForType() { "als.proto\"v\n\034CreateUploadLocationRespons" + "e\022\022\n\nsigned_url\030\001 \001(\t\022\022\n\nnative_url\030\002 \001(" + "\t\022.\n\nexpires_at\030\003 \001(\0132\032.google.protobuf." + - "Timestamp\"\224\001\n\033CreateUploadLocationReques" + + "Timestamp\"\253\001\n\033CreateUploadLocationReques" + "t\022\017\n\007project\030\001 \001(\t\022\016\n\006domain\030\002 \001(\t\022\020\n\010fi" + "lename\030\003 \001(\t\022-\n\nexpires_in\030\004 \001(\0132\031.googl" + "e.protobuf.Duration\022\023\n\013content_md5\030\005 \001(\014" + - "\"f\n\035CreateDownloadLocationRequest\022\022\n\nnat" + - "ive_url\030\001 \001(\t\022-\n\nexpires_in\030\002 \001(\0132\031.goog" + - "le.protobuf.Duration:\002\030\001\"h\n\036CreateDownlo" + - "adLocationResponse\022\022\n\nsigned_url\030\001 \001(\t\022." + - "\n\nexpires_at\030\002 \001(\0132\032.google.protobuf.Tim" + - "estamp:\002\030\001\"\320\001\n\031CreateDownloadLinkRequest" + - "\0225\n\rartifact_type\030\001 \001(\0162\036.flyteidl.servi" + - "ce.ArtifactType\022-\n\nexpires_in\030\002 \001(\0132\031.go" + - "ogle.protobuf.Duration\022C\n\021node_execution" + - "_id\030\003 \001(\0132&.flyteidl.core.NodeExecutionI" + - "dentifierH\000B\010\n\006source\"\242\001\n\032CreateDownload" + - "LinkResponse\022\026\n\nsigned_url\030\001 \003(\tB\002\030\001\0222\n\n" + - "expires_at\030\002 \001(\0132\032.google.protobuf.Times" + - "tampB\002\030\001\0228\n\017pre_signed_urls\030\003 \001(\0132\037.flyt" + - "eidl.service.PreSignedURLs\"S\n\rPreSignedU" + - "RLs\022\022\n\nsigned_url\030\001 \003(\t\022.\n\nexpires_at\030\002 " + - "\001(\0132\032.google.protobuf.Timestamp\"#\n\016GetDa" + - "taRequest\022\021\n\tflyte_url\030\001 \001(\t\"\262\001\n\017GetData" + - "Response\0220\n\013literal_map\030\001 \001(\0132\031.flyteidl" + - ".core.LiteralMapH\000\022:\n\017pre_signed_urls\030\002 " + - "\001(\0132\037.flyteidl.service.PreSignedURLsH\000\022)" + - "\n\007literal\030\003 \001(\0132\026.flyteidl.core.LiteralH" + - "\000B\006\n\004data*C\n\014ArtifactType\022\033\n\027ARTIFACT_TY" + - "PE_UNDEFINED\020\000\022\026\n\022ARTIFACT_TYPE_DECK\020\0012\342" + - "\004\n\020DataProxyService\022\240\001\n\024CreateUploadLoca" + - "tion\022-.flyteidl.service.CreateUploadLoca" + - "tionRequest\032..flyteidl.service.CreateUpl" + - "oadLocationResponse\")\202\323\344\223\002#\"\036/api/v1/dat" + - "aproxy/artifact_urn:\001*\022\246\001\n\026CreateDownloa" + - "dLocation\022/.flyteidl.service.CreateDownl" + - "oadLocationRequest\0320.flyteidl.service.Cr" + - "eateDownloadLocationResponse\")\210\002\001\202\323\344\223\002 \022" + - "\036/api/v1/dataproxy/artifact_urn\022\233\001\n\022Crea" + - "teDownloadLink\022+.flyteidl.service.Create" + - "DownloadLinkRequest\032,.flyteidl.service.C" + - "reateDownloadLinkResponse\"*\202\323\344\223\002$\"\037/api/" + - "v1/dataproxy/artifact_link:\001*\022d\n\007GetData" + - "\022 .flyteidl.service.GetDataRequest\032!.fly" + - "teidl.service.GetDataResponse\"\024\202\323\344\223\002\016\022\014/" + - "api/v1/dataB9Z7github.com/flyteorg/flyte" + - "idl/gen/pb-go/flyteidl/serviceb\006proto3" + "\022\025\n\rfilename_root\030\006 \001(\t\"f\n\035CreateDownloa" + + "dLocationRequest\022\022\n\nnative_url\030\001 \001(\t\022-\n\n" + + "expires_in\030\002 \001(\0132\031.google.protobuf.Durat" + + "ion:\002\030\001\"h\n\036CreateDownloadLocationRespons" + + "e\022\022\n\nsigned_url\030\001 \001(\t\022.\n\nexpires_at\030\002 \001(" + + "\0132\032.google.protobuf.Timestamp:\002\030\001\"\320\001\n\031Cr" + + "eateDownloadLinkRequest\0225\n\rartifact_type" + + "\030\001 \001(\0162\036.flyteidl.service.ArtifactType\022-" + + "\n\nexpires_in\030\002 \001(\0132\031.google.protobuf.Dur" + + "ation\022C\n\021node_execution_id\030\003 \001(\0132&.flyte" + + "idl.core.NodeExecutionIdentifierH\000B\010\n\006so" + + "urce\"\242\001\n\032CreateDownloadLinkResponse\022\026\n\ns" + + "igned_url\030\001 \003(\tB\002\030\001\0222\n\nexpires_at\030\002 \001(\0132" + + "\032.google.protobuf.TimestampB\002\030\001\0228\n\017pre_s" + + "igned_urls\030\003 \001(\0132\037.flyteidl.service.PreS" + + "ignedURLs\"S\n\rPreSignedURLs\022\022\n\nsigned_url" + + "\030\001 \003(\t\022.\n\nexpires_at\030\002 \001(\0132\032.google.prot" + + "obuf.Timestamp\"#\n\016GetDataRequest\022\021\n\tflyt" + + "e_url\030\001 \001(\t\"\262\001\n\017GetDataResponse\0220\n\013liter" + + "al_map\030\001 \001(\0132\031.flyteidl.core.LiteralMapH" + + "\000\022:\n\017pre_signed_urls\030\002 \001(\0132\037.flyteidl.se" + + "rvice.PreSignedURLsH\000\022)\n\007literal\030\003 \001(\0132\026" + + ".flyteidl.core.LiteralH\000B\006\n\004data*C\n\014Arti" + + "factType\022\033\n\027ARTIFACT_TYPE_UNDEFINED\020\000\022\026\n" + + "\022ARTIFACT_TYPE_DECK\020\0012\342\004\n\020DataProxyServi" + + "ce\022\240\001\n\024CreateUploadLocation\022-.flyteidl.s" + + "ervice.CreateUploadLocationRequest\032..fly" + + "teidl.service.CreateUploadLocationRespon" + + "se\")\202\323\344\223\002#\"\036/api/v1/dataproxy/artifact_u" + + "rn:\001*\022\246\001\n\026CreateDownloadLocation\022/.flyte" + + "idl.service.CreateDownloadLocationReques" + + "t\0320.flyteidl.service.CreateDownloadLocat" + + "ionResponse\")\210\002\001\202\323\344\223\002 \022\036/api/v1/dataprox" + + "y/artifact_urn\022\233\001\n\022CreateDownloadLink\022+." + + "flyteidl.service.CreateDownloadLinkReque" + + "st\032,.flyteidl.service.CreateDownloadLink" + + "Response\"*\202\323\344\223\002$\"\037/api/v1/dataproxy/arti" + + "fact_link:\001*\022d\n\007GetData\022 .flyteidl.servi" + + "ce.GetDataRequest\032!.flyteidl.service.Get" + + "DataResponse\"\024\202\323\344\223\002\016\022\014/api/v1/dataB9Z7gi" + + "thub.com/flyteorg/flyteidl/gen/pb-go/fly" + + "teidl/serviceb\006proto3" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { @@ -9619,7 +9828,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_flyteidl_service_CreateUploadLocationRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_flyteidl_service_CreateUploadLocationRequest_descriptor, - new java.lang.String[] { "Project", "Domain", "Filename", "ExpiresIn", "ContentMd5", }); + new java.lang.String[] { "Project", "Domain", "Filename", "ExpiresIn", "ContentMd5", "FilenameRoot", }); internal_static_flyteidl_service_CreateDownloadLocationRequest_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_flyteidl_service_CreateDownloadLocationRequest_fieldAccessorTable = new diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index d746cc6fe..26ead72f4 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -19686,6 +19686,9 @@ export namespace flyteidl { /** CreateUploadLocationRequest contentMd5 */ contentMd5?: (Uint8Array|null); + + /** CreateUploadLocationRequest filenameRoot */ + filenameRoot?: (string|null); } /** Represents a CreateUploadLocationRequest. */ @@ -19712,6 +19715,9 @@ export namespace flyteidl { /** CreateUploadLocationRequest contentMd5. */ public contentMd5: Uint8Array; + /** CreateUploadLocationRequest filenameRoot. */ + public filenameRoot: string; + /** * Creates a new CreateUploadLocationRequest instance using the specified properties. * @param [properties] Properties to set diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index f52945631..ff60b3983 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -45833,6 +45833,7 @@ * @property {string|null} [filename] CreateUploadLocationRequest filename * @property {google.protobuf.IDuration|null} [expiresIn] CreateUploadLocationRequest expiresIn * @property {Uint8Array|null} [contentMd5] CreateUploadLocationRequest contentMd5 + * @property {string|null} [filenameRoot] CreateUploadLocationRequest filenameRoot */ /** @@ -45890,6 +45891,14 @@ */ CreateUploadLocationRequest.prototype.contentMd5 = $util.newBuffer([]); + /** + * CreateUploadLocationRequest filenameRoot. + * @member {string} filenameRoot + * @memberof flyteidl.service.CreateUploadLocationRequest + * @instance + */ + CreateUploadLocationRequest.prototype.filenameRoot = ""; + /** * Creates a new CreateUploadLocationRequest instance using the specified properties. * @function create @@ -45924,6 +45933,8 @@ $root.google.protobuf.Duration.encode(message.expiresIn, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); if (message.contentMd5 != null && message.hasOwnProperty("contentMd5")) writer.uint32(/* id 5, wireType 2 =*/42).bytes(message.contentMd5); + if (message.filenameRoot != null && message.hasOwnProperty("filenameRoot")) + writer.uint32(/* id 6, wireType 2 =*/50).string(message.filenameRoot); return writer; }; @@ -45960,6 +45971,9 @@ case 5: message.contentMd5 = reader.bytes(); break; + case 6: + message.filenameRoot = reader.string(); + break; default: reader.skipType(tag & 7); break; @@ -45996,6 +46010,9 @@ if (message.contentMd5 != null && message.hasOwnProperty("contentMd5")) if (!(message.contentMd5 && typeof message.contentMd5.length === "number" || $util.isString(message.contentMd5))) return "contentMd5: buffer expected"; + if (message.filenameRoot != null && message.hasOwnProperty("filenameRoot")) + if (!$util.isString(message.filenameRoot)) + return "filenameRoot: string expected"; return null; }; diff --git a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py index 6ab9391e6..a8f41072b 100644 --- a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py @@ -18,7 +18,7 @@ from flyteidl.core import literals_pb2 as flyteidl_dot_core_dot_literals__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n flyteidl/service/dataproxy.proto\x12\x10\x66lyteidl.service\x1a\x1cgoogle/api/annotations.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1c\x66lyteidl/core/literals.proto\"\x97\x01\n\x1c\x43reateUploadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x1d\n\nnative_url\x18\x02 \x01(\tR\tnativeUrl\x12\x39\n\nexpires_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\"\xc6\x01\n\x1b\x43reateUploadLocationRequest\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x16\n\x06\x64omain\x18\x02 \x01(\tR\x06\x64omain\x12\x1a\n\x08\x66ilename\x18\x03 \x01(\tR\x08\x66ilename\x12\x38\n\nexpires_in\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12\x1f\n\x0b\x63ontent_md5\x18\x05 \x01(\x0cR\ncontentMd5\"|\n\x1d\x43reateDownloadLocationRequest\x12\x1d\n\nnative_url\x18\x01 \x01(\tR\tnativeUrl\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn:\x02\x18\x01\"~\n\x1e\x43reateDownloadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt:\x02\x18\x01\"\xfa\x01\n\x19\x43reateDownloadLinkRequest\x12\x43\n\rartifact_type\x18\x01 \x01(\x0e\x32\x1e.flyteidl.service.ArtifactTypeR\x0c\x61rtifactType\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12T\n\x11node_execution_id\x18\x03 \x01(\x0b\x32&.flyteidl.core.NodeExecutionIdentifierH\x00R\x0fnodeExecutionIdB\x08\n\x06source\"\xc7\x01\n\x1a\x43reateDownloadLinkResponse\x12!\n\nsigned_url\x18\x01 \x03(\tB\x02\x18\x01R\tsignedUrl\x12=\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01R\texpiresAt\x12G\n\x0fpre_signed_urls\x18\x03 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsR\rpreSignedUrls\"i\n\rPreSignedURLs\x12\x1d\n\nsigned_url\x18\x01 \x03(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\"-\n\x0eGetDataRequest\x12\x1b\n\tflyte_url\x18\x01 \x01(\tR\x08\x66lyteUrl\"\xd6\x01\n\x0fGetDataResponse\x12<\n\x0bliteral_map\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapH\x00R\nliteralMap\x12I\n\x0fpre_signed_urls\x18\x02 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsH\x00R\rpreSignedUrls\x12\x32\n\x07literal\x18\x03 \x01(\x0b\x32\x16.flyteidl.core.LiteralH\x00R\x07literalB\x06\n\x04\x64\x61ta*C\n\x0c\x41rtifactType\x12\x1b\n\x17\x41RTIFACT_TYPE_UNDEFINED\x10\x00\x12\x16\n\x12\x41RTIFACT_TYPE_DECK\x10\x01\x32\xe2\x04\n\x10\x44\x61taProxyService\x12\xa0\x01\n\x14\x43reateUploadLocation\x12-.flyteidl.service.CreateUploadLocationRequest\x1a..flyteidl.service.CreateUploadLocationResponse\")\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/api/v1/dataproxy/artifact_urn\x12\xa6\x01\n\x16\x43reateDownloadLocation\x12/.flyteidl.service.CreateDownloadLocationRequest\x1a\x30.flyteidl.service.CreateDownloadLocationResponse\")\x88\x02\x01\x82\xd3\xe4\x93\x02 \x12\x1e/api/v1/dataproxy/artifact_urn\x12\x9b\x01\n\x12\x43reateDownloadLink\x12+.flyteidl.service.CreateDownloadLinkRequest\x1a,.flyteidl.service.CreateDownloadLinkResponse\"*\x82\xd3\xe4\x93\x02$:\x01*\"\x1f/api/v1/dataproxy/artifact_link\x12\x64\n\x07GetData\x12 .flyteidl.service.GetDataRequest\x1a!.flyteidl.service.GetDataResponse\"\x14\x82\xd3\xe4\x93\x02\x0e\x12\x0c/api/v1/dataB\xc0\x01\n\x14\x63om.flyteidl.serviceB\x0e\x44\x61taproxyProtoP\x01Z7github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service\xa2\x02\x03\x46SX\xaa\x02\x10\x46lyteidl.Service\xca\x02\x10\x46lyteidl\\Service\xe2\x02\x1c\x46lyteidl\\Service\\GPBMetadata\xea\x02\x11\x46lyteidl::Serviceb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n flyteidl/service/dataproxy.proto\x12\x10\x66lyteidl.service\x1a\x1cgoogle/api/annotations.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1c\x66lyteidl/core/literals.proto\"\x97\x01\n\x1c\x43reateUploadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x1d\n\nnative_url\x18\x02 \x01(\tR\tnativeUrl\x12\x39\n\nexpires_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\"\xeb\x01\n\x1b\x43reateUploadLocationRequest\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x16\n\x06\x64omain\x18\x02 \x01(\tR\x06\x64omain\x12\x1a\n\x08\x66ilename\x18\x03 \x01(\tR\x08\x66ilename\x12\x38\n\nexpires_in\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12\x1f\n\x0b\x63ontent_md5\x18\x05 \x01(\x0cR\ncontentMd5\x12#\n\rfilename_root\x18\x06 \x01(\tR\x0c\x66ilenameRoot\"|\n\x1d\x43reateDownloadLocationRequest\x12\x1d\n\nnative_url\x18\x01 \x01(\tR\tnativeUrl\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn:\x02\x18\x01\"~\n\x1e\x43reateDownloadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt:\x02\x18\x01\"\xfa\x01\n\x19\x43reateDownloadLinkRequest\x12\x43\n\rartifact_type\x18\x01 \x01(\x0e\x32\x1e.flyteidl.service.ArtifactTypeR\x0c\x61rtifactType\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12T\n\x11node_execution_id\x18\x03 \x01(\x0b\x32&.flyteidl.core.NodeExecutionIdentifierH\x00R\x0fnodeExecutionIdB\x08\n\x06source\"\xc7\x01\n\x1a\x43reateDownloadLinkResponse\x12!\n\nsigned_url\x18\x01 \x03(\tB\x02\x18\x01R\tsignedUrl\x12=\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01R\texpiresAt\x12G\n\x0fpre_signed_urls\x18\x03 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsR\rpreSignedUrls\"i\n\rPreSignedURLs\x12\x1d\n\nsigned_url\x18\x01 \x03(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\"-\n\x0eGetDataRequest\x12\x1b\n\tflyte_url\x18\x01 \x01(\tR\x08\x66lyteUrl\"\xd6\x01\n\x0fGetDataResponse\x12<\n\x0bliteral_map\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapH\x00R\nliteralMap\x12I\n\x0fpre_signed_urls\x18\x02 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsH\x00R\rpreSignedUrls\x12\x32\n\x07literal\x18\x03 \x01(\x0b\x32\x16.flyteidl.core.LiteralH\x00R\x07literalB\x06\n\x04\x64\x61ta*C\n\x0c\x41rtifactType\x12\x1b\n\x17\x41RTIFACT_TYPE_UNDEFINED\x10\x00\x12\x16\n\x12\x41RTIFACT_TYPE_DECK\x10\x01\x32\xe2\x04\n\x10\x44\x61taProxyService\x12\xa0\x01\n\x14\x43reateUploadLocation\x12-.flyteidl.service.CreateUploadLocationRequest\x1a..flyteidl.service.CreateUploadLocationResponse\")\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/api/v1/dataproxy/artifact_urn\x12\xa6\x01\n\x16\x43reateDownloadLocation\x12/.flyteidl.service.CreateDownloadLocationRequest\x1a\x30.flyteidl.service.CreateDownloadLocationResponse\")\x88\x02\x01\x82\xd3\xe4\x93\x02 \x12\x1e/api/v1/dataproxy/artifact_urn\x12\x9b\x01\n\x12\x43reateDownloadLink\x12+.flyteidl.service.CreateDownloadLinkRequest\x1a,.flyteidl.service.CreateDownloadLinkResponse\"*\x82\xd3\xe4\x93\x02$:\x01*\"\x1f/api/v1/dataproxy/artifact_link\x12\x64\n\x07GetData\x12 .flyteidl.service.GetDataRequest\x1a!.flyteidl.service.GetDataResponse\"\x14\x82\xd3\xe4\x93\x02\x0e\x12\x0c/api/v1/dataB\xc0\x01\n\x14\x63om.flyteidl.serviceB\x0e\x44\x61taproxyProtoP\x01Z7github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/service\xa2\x02\x03\x46SX\xaa\x02\x10\x46lyteidl.Service\xca\x02\x10\x46lyteidl\\Service\xe2\x02\x1c\x46lyteidl\\Service\\GPBMetadata\xea\x02\x11\x46lyteidl::Serviceb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -43,26 +43,26 @@ _DATAPROXYSERVICE.methods_by_name['CreateDownloadLink']._serialized_options = b'\202\323\344\223\002$:\001*\"\037/api/v1/dataproxy/artifact_link' _DATAPROXYSERVICE.methods_by_name['GetData']._options = None _DATAPROXYSERVICE.methods_by_name['GetData']._serialized_options = b'\202\323\344\223\002\016\022\014/api/v1/data' - _globals['_ARTIFACTTYPE']._serialized_start=1646 - _globals['_ARTIFACTTYPE']._serialized_end=1713 + _globals['_ARTIFACTTYPE']._serialized_start=1683 + _globals['_ARTIFACTTYPE']._serialized_end=1750 _globals['_CREATEUPLOADLOCATIONRESPONSE']._serialized_start=212 _globals['_CREATEUPLOADLOCATIONRESPONSE']._serialized_end=363 _globals['_CREATEUPLOADLOCATIONREQUEST']._serialized_start=366 - _globals['_CREATEUPLOADLOCATIONREQUEST']._serialized_end=564 - _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_start=566 - _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_end=690 - _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_start=692 - _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_end=818 - _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_start=821 - _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_end=1071 - _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_start=1074 - _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_end=1273 - _globals['_PRESIGNEDURLS']._serialized_start=1275 - _globals['_PRESIGNEDURLS']._serialized_end=1380 - _globals['_GETDATAREQUEST']._serialized_start=1382 - _globals['_GETDATAREQUEST']._serialized_end=1427 - _globals['_GETDATARESPONSE']._serialized_start=1430 - _globals['_GETDATARESPONSE']._serialized_end=1644 - _globals['_DATAPROXYSERVICE']._serialized_start=1716 - _globals['_DATAPROXYSERVICE']._serialized_end=2326 + _globals['_CREATEUPLOADLOCATIONREQUEST']._serialized_end=601 + _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_start=603 + _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_end=727 + _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_start=729 + _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_end=855 + _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_start=858 + _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_end=1108 + _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_start=1111 + _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_end=1310 + _globals['_PRESIGNEDURLS']._serialized_start=1312 + _globals['_PRESIGNEDURLS']._serialized_end=1417 + _globals['_GETDATAREQUEST']._serialized_start=1419 + _globals['_GETDATAREQUEST']._serialized_end=1464 + _globals['_GETDATARESPONSE']._serialized_start=1467 + _globals['_GETDATARESPONSE']._serialized_end=1681 + _globals['_DATAPROXYSERVICE']._serialized_start=1753 + _globals['_DATAPROXYSERVICE']._serialized_end=2363 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi index badf93316..820034751 100644 --- a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi @@ -29,18 +29,20 @@ class CreateUploadLocationResponse(_message.Message): def __init__(self, signed_url: _Optional[str] = ..., native_url: _Optional[str] = ..., expires_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ... class CreateUploadLocationRequest(_message.Message): - __slots__ = ["project", "domain", "filename", "expires_in", "content_md5"] + __slots__ = ["project", "domain", "filename", "expires_in", "content_md5", "filename_root"] PROJECT_FIELD_NUMBER: _ClassVar[int] DOMAIN_FIELD_NUMBER: _ClassVar[int] FILENAME_FIELD_NUMBER: _ClassVar[int] EXPIRES_IN_FIELD_NUMBER: _ClassVar[int] CONTENT_MD5_FIELD_NUMBER: _ClassVar[int] + FILENAME_ROOT_FIELD_NUMBER: _ClassVar[int] project: str domain: str filename: str expires_in: _duration_pb2.Duration content_md5: bytes - def __init__(self, project: _Optional[str] = ..., domain: _Optional[str] = ..., filename: _Optional[str] = ..., expires_in: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., content_md5: _Optional[bytes] = ...) -> None: ... + filename_root: str + def __init__(self, project: _Optional[str] = ..., domain: _Optional[str] = ..., filename: _Optional[str] = ..., expires_in: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., content_md5: _Optional[bytes] = ..., filename_root: _Optional[str] = ...) -> None: ... class CreateDownloadLocationRequest(_message.Message): __slots__ = ["native_url", "expires_in"] diff --git a/flyteidl/gen/pb_rust/flyteidl.service.rs b/flyteidl/gen/pb_rust/flyteidl.service.rs index d7b7ee83a..197e99055 100644 --- a/flyteidl/gen/pb_rust/flyteidl.service.rs +++ b/flyteidl/gen/pb_rust/flyteidl.service.rs @@ -87,6 +87,10 @@ pub struct CreateUploadLocationResponse { pub expires_at: ::core::option::Option<::prost_types::Timestamp>, } /// CreateUploadLocationRequest specified request for the CreateUploadLocation API. +/// The implementation in data proxy service will create the s3 location with some server side configured prefixes, +/// and then: +/// - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR +/// - project/domain/filename_root (if present)/filename (if present). #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CreateUploadLocationRequest { @@ -112,6 +116,12 @@ pub struct CreateUploadLocationRequest { /// +required #[prost(bytes="vec", tag="5")] pub content_md5: ::prost::alloc::vec::Vec, + /// If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included + /// this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix + /// in data proxy config. This option is useful when uploading multiple files. + /// +optional + #[prost(string, tag="6")] + pub filename_root: ::prost::alloc::string::String, } /// CreateDownloadLocationRequest specified request for the CreateDownloadLocation API. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/flyteidl/protos/flyteidl/service/dataproxy.proto b/flyteidl/protos/flyteidl/service/dataproxy.proto index eac82aa5a..8972d4f6d 100644 --- a/flyteidl/protos/flyteidl/service/dataproxy.proto +++ b/flyteidl/protos/flyteidl/service/dataproxy.proto @@ -23,6 +23,10 @@ message CreateUploadLocationResponse { } // CreateUploadLocationRequest specified request for the CreateUploadLocation API. +// The implementation in data proxy service will create the s3 location with some server side configured prefixes, +// and then: +// - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR +// - project/domain/filename_root (if present)/filename (if present). message CreateUploadLocationRequest { // Project to create the upload location for // +required @@ -45,6 +49,12 @@ message CreateUploadLocationRequest { // generated path. // +required bytes content_md5 = 5; + + // If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included + // this makes the upload location deterministic. The native url will still be prefixed by the upload location prefix + // in data proxy config. This option is useful when uploading multiple files. + // +optional + string filename_root = 6; } // CreateDownloadLocationRequest specified request for the CreateDownloadLocation API.