From 8396b1c541023b384f0df5a6f9391866242024a2 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Fri, 2 Aug 2024 09:09:00 -0700 Subject: [PATCH] Added an endpoint allowing file metadata to be requested by ID. --- services/prototype.go | 52 ++++++++++++++++++++++++++++++++++++ services/prototype_test.go | 29 ++++++++++++++++++++ services/transfer_service.go | 35 +++++++----------------- 3 files changed, 91 insertions(+), 25 deletions(-) diff --git a/services/prototype.go b/services/prototype.go index 516f881b..e9b9ec48 100644 --- a/services/prototype.go +++ b/services/prototype.go @@ -389,6 +389,57 @@ func (service *prototype) searchDatabaseWithSpecificParams(ctx context.Context, return searchDatabase(ctx, &searchInput, body.Specific) } +type FileMetadataOutput struct { + Body FileMetadataResponse `doc:"Metadata for files with the given IDs"` +} + +// fetches file metadata given a list of file identifiers +func (service *prototype) fetchFileMetadata(ctx context.Context, + input *struct { + Authorization string `header:"authorization" doc:"Authorization header with encoded access token"` + Database string `json:"database" query:"database" example:"jdp" doc:"The ID of the database for which file metadata is fetched"` + Ids string `json:"ids" query:"ids" example:"JDP:6101cc0f2b1f2eeea564c978" doc:"A comma-separated list of file IDs"` + Offset int `json:"offset" query:"offset" example:"100" doc:"Metadata records begin at the given offset"` + Limit int `json:"limit" query:"limit" example:"50" doc:"Limits the number of metadata records returned"` + }) (*FileMetadataOutput, error) { + + orcid, err := authorize(input.Authorization) + if err != nil { + return nil, err + } + + // is the database valid? + _, ok := config.Databases[input.Database] + if !ok { + return nil, fmt.Errorf("Database %s not found", input.Database) + } + + // have we been given any IDs? + if strings.TrimSpace(input.Ids) == "" { + return nil, huma.Error400BadRequest("No file IDs were provided!") + } + ids := strings.Split(input.Ids, ",") + + slog.Info(fmt.Sprintf("Fetching file metadata for %d files in database %s...", + len(ids), input.Database)) + db, err := databases.NewDatabase(orcid, input.Database) + if err != nil { + return nil, err + } + + results, err := db.Resources(ids) + if err != nil { + slog.Error(err.Error()) + return nil, err + } + return &FileMetadataOutput{ + Body: FileMetadataResponse{ + Database: input.Database, + Resources: results, + }, + }, nil +} + type TransferOutput struct { Body TransferResponse `doc:"A UUID for the requested transfer"` Status int @@ -527,6 +578,7 @@ func NewDTSPrototype() (TransferService, error) { huma.Get(api, "/api/v1/databases/{db}/search-parameters", service.getDatabaseSearchParameters) huma.Get(api, "/api/v1/files", service.searchDatabase) huma.Post(api, "/api/v1/files", service.searchDatabaseWithSpecificParams) + huma.Get(api, "/api/v1/files/by-id", service.fetchFileMetadata) huma.Post(api, "/api/v1/transfers", service.createTransfer) huma.Get(api, "/api/v1/transfers/{id}", service.getTransferStatus) huma.Delete(api, "/api/v1/transfers/{id}", service.deleteTransfer) diff --git a/services/prototype_test.go b/services/prototype_test.go index c09955d9..6ca236c4 100644 --- a/services/prototype_test.go +++ b/services/prototype_test.go @@ -416,6 +416,35 @@ func TestSearchDatabase(t *testing.T) { assert.Equal("file1", results.Resources[0].Name) } +// fetches file metadata from the JDP for some specific files +func TestFetchJdpMetadata(t *testing.T) { + assert := assert.New(t) + + // try omitting file IDs + resp, err := get(baseUrl + apiPrefix + "files/by-id?database=jdp") + assert.Nil(err) + assert.Equal(http.StatusBadRequest, resp.StatusCode) + + // now let's fetch 3 records + resp, err = get(baseUrl + apiPrefix + + "files/by-id?database=jdp&ids=JDP:6101cc0f2b1f2eeea564c978,JDP:613a7baa72d3a08c9a54b32d,JDP:61412246cc4ff44f36c8913d") + assert.Nil(err) + + respBody, err := io.ReadAll(resp.Body) + assert.Nil(err) + assert.Equal(http.StatusOK, resp.StatusCode) + defer resp.Body.Close() + + var results SearchResultsResponse + err = json.Unmarshal(respBody, &results) + assert.Nil(err) + assert.Equal("jdp", results.Database) + assert.Equal(3, len(results.Resources)) + assert.Equal("JDP:6101cc0f2b1f2eeea564c978", results.Resources[0].Id) + assert.Equal("JDP:613a7baa72d3a08c9a54b32d", results.Resources[1].Id) + assert.Equal("JDP:61412246cc4ff44f36c8913d", results.Resources[2].Id) +} + // creates a transfer from source -> destination1 func TestCreateTransfer(t *testing.T) { assert := assert.New(t) diff --git a/services/transfer_service.go b/services/transfer_service.go index 3966043f..eb142cbc 100644 --- a/services/transfer_service.go +++ b/services/transfer_service.go @@ -2,24 +2,12 @@ package services import ( "context" - "encoding/json" - "net/http" "github.com/google/uuid" "github.com/kbase/dts/frictionless" ) -// This package-specific helper function writes a JSON payload to an -// http.ResponseWriter. -func writeJson(w http.ResponseWriter, data []byte, code int) { - w.WriteHeader(code) - if len(data) > 0 { - w.Header().Set("Content-Type", "application/json") - w.Write(data) - } -} - // this type encodes a JSON object for responding to root queries type ServiceInfoResponse struct { Name string `json:"name" example:"DTS" doc:"The name of the service API"` @@ -37,17 +25,6 @@ type ErrorResponse struct { Error string `json:"message"` } -// This package-specific helper function writes an error to an -// http.ResponseWriter, giving it the proper status code, and encoding an -// ErrorResponse in the response body. -func writeError(w http.ResponseWriter, message string, code int) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(code) - e := ErrorResponse{Code: code, Error: message} - data, _ := json.Marshal(e) - w.Write(data) -} - // a response for a database-related query (GET) type DatabaseResponse struct { Id string `json:"id" example:"jdp" ` @@ -56,13 +33,21 @@ type DatabaseResponse struct { URL string `json:"url" example:"https://data.jgi.doe.gov"` } -// a response for a query (GET) +// a response for a file search query (GET) type SearchResultsResponse struct { // name of organization database Database string `json:"database" example:"jdp" doc:"the database searched"` // ElasticSearch query string Query string `json:"query" example:"prochlorococcus" doc:"the given query string"` - // Resources matching the query + // resources matching the query + Resources []frictionless.DataResource `json:"resources" doc:"an array of Frictionless DataResources"` +} + +// a response for a file metadata query (GET) +type FileMetadataResponse struct { + // name of organization database + Database string `json:"database" example:"jdp" doc:"the database searched"` + // resources corresponding to given file IDs Resources []frictionless.DataResource `json:"resources" doc:"an array of Frictionless DataResources"` }