feat: add file download and size to remaining entities (#92) (#93)

* feat: update sequencing-data.csv to have read file_size from S3 (#92)

* refactor: adjust build-sequencing-data and buildHelp (#92)

* style: adjust python file names to be more conventional (#92)

* style: change names in python files to be more conventional (#92)

* feat: add file size to annotation and assembly csvs (#92)

* feat: add file size and name to all entities (#92)

* feat: update alignments.json (#92)

* feat: add download, filename, and file size columns to all entities (#92)

---------

Co-authored-by: hunterckx <[email protected]>

app/apis/catalog/hprc-data-explorer/common/entities.ts

@@ -1,3 +1,9 @@
+export type HPRCDataExplorerEntity =
+  | HPRCDataExplorerAlignment
+  | HPRCDataExplorerAnnotation
+  | HPRCDataExplorerAssembly
+  | HPRCDataExplorerRawSequencingData;
+
 export interface HPRCDataExplorerRawSequencingData {
   accession: string | null;
   assembly: string | null;
@@ -13,6 +19,7 @@ export interface HPRCDataExplorerRawSequencingData {
   designDescription: string;
   familyId: string | null;
   filename: string;
+  fileSize: string; // This and other fileSize fields should be `number | LABEL.NA` if that's ever restored
   filetype: string;
   fiveHundredkbPlus: string; //number | LABEL.NA;
   fourHundredkbPlus: string; //number | LABEL.NA;
@@ -69,6 +76,8 @@ export interface HPRCDataExplorerAssembly {
   awsFasta: string | null;
   familyId: string | null;
   fastaSha256: string | null;
+  filename: string;
+  fileSize: string;
   frag: number | null;
   fullDup: number | null;
   fullSgl: number | null;
@@ -90,6 +99,8 @@ export interface HPRCDataExplorerAssembly {
 export interface HPRCDataExplorerAnnotation {
   annotationType: string;
   fileLocation: string;
+  filename: string;
+  fileSize: string;
   haplotype: string | null;
   reference: string | null;
   sampleId: string;

app/viewModelBuilders/catalog/hprc-data-explorer/common/viewModelBuilders.ts

@@ -3,7 +3,9 @@ import {
   HPRCDataExplorerAlignment,
   HPRCDataExplorerAnnotation,
   HPRCDataExplorerAssembly,
+  HPRCDataExplorerEntity,
   HPRCDataExplorerRawSequencingData,
+  LABEL,
 } from "../../../../apis/catalog/hprc-data-explorer/common/entities";
 import * as C from "../../../../components/index";
 import { METADATA_KEY } from "./entities";
@@ -45,7 +47,21 @@ export const buildAlignmentDownload = (
 ): React.ComponentProps<typeof C.FileDownload> => {
   return {
     fileName: alignment.filename,
-    fileUrl: getAlignmentDownloadUrl(alignment),
+    fileUrl: getDownloadUrl(alignment.loc),
+  };
+};
+
+/**
+ * Build props for the annotation download cell.
+ * @param annotation - Annotation entity.
+ * @returns Props to be used for the cell.
+ */
+export const buildAnnotationDownload = (
+  annotation: HPRCDataExplorerAnnotation
+): React.ComponentProps<typeof C.FileDownload> => {
+  return {
+    fileName: annotation.filename,
+    fileUrl: getDownloadUrl(annotation.fileLocation),
   };
 };
 
@@ -75,6 +91,20 @@ export const buildAssembly = (
   };
 };
 
+/**
+ * Build props for the assembly download cell.
+ * @param assembly - Assembly entity.
+ * @returns Props to be used for the cell.
+ */
+export const buildAssemblyDownload = (
+  assembly: HPRCDataExplorerAssembly
+): React.ComponentProps<typeof C.FileDownload> => {
+  return {
+    fileName: assembly.filename,
+    fileUrl: assembly.awsFasta ? getDownloadUrl(assembly.awsFasta) : undefined,
+  };
+};
+
 /**
  * Build props for the AWS FASTA cell.
  * @param assembly - Assembly entity.
@@ -259,11 +289,11 @@ export const buildFileLocation = (
 
 /**
  * Build props for the filename cell.
- * @param entity - Raw sequencing data or alignment entity.
+ * @param entity - Entity.
  * @returns Props to be used for the cell.
  */
 export const buildFilename = (
-  entity: HPRCDataExplorerRawSequencingData | HPRCDataExplorerAlignment
+  entity: HPRCDataExplorerEntity
 ): React.ComponentProps<typeof C.BasicCell> => {
   return {
     value: entity.filename,
@@ -272,14 +302,14 @@ export const buildFilename = (
 
 /**
  * Build props for the file size cell.
- * @param alignment - Alignment entity.
+ * @param entity - Entity.
  * @returns Props to be used for the cell.
  */
 export const buildFileSize = (
-  alignment: HPRCDataExplorerAlignment
+  entity: HPRCDataExplorerEntity
 ): React.ComponentProps<typeof C.BasicCell> => {
   return {
-    value: formatFileSize(alignment.fileSize),
+    value: entity.fileSize === LABEL.NA ? LABEL.NA : formatFileSize(Number(entity.fileSize)),
   };
 };
 
@@ -900,6 +930,20 @@ export const buildSampleId = (
   };
 };
 
+/**
+ * Build props for the sequencing data download cell.
+ * @param rawSequencingData - Raw sequencing data entity.
+ * @returns Props to be used for the cell.
+ */
+export const buildSequencingDataDownload = (
+  rawSequencingData: HPRCDataExplorerRawSequencingData
+): React.ComponentProps<typeof C.FileDownload> => {
+  return {
+    fileName: rawSequencingData.filename,
+    fileUrl: getDownloadUrl(rawSequencingData.path),
+  };
+};
+
 /**
  * Build props for the seq kit cell.
  * @param rawSequencingData - Raw sequencing data entity.
@@ -1148,13 +1192,13 @@ function formatPercentage(decimalFraction: number): string {
 }
 
 /**
- * Get the download URL for an alignment.
- * @param alignment - Alignment entity.
+ * Get a download URL from a given URI, by converting it from an S3 URI or returning it as-is.
+ * @param uri - URI.
  * @returns download URL.
  */
-function getAlignmentDownloadUrl(alignment: HPRCDataExplorerAlignment): string {
-  const s3UriMatch = /^s3:\/\/([^/]+)\/(.*)$/.exec(alignment.loc);
-  if (!s3UriMatch) return alignment.loc;
+function getDownloadUrl(uri: string): string {
+  const s3UriMatch = /^s3:\/\/([^/]+)\/(.*)$/.exec(uri);
+  if (!s3UriMatch) return uri;
   const [, bucketName, filePath] = s3UriMatch;
   return `https://${bucketName}.s3.amazonaws.com/${filePath}`;
 }

catalog-build/README.md

@@ -14,9 +14,9 @@ pip install -r catalog-build/requirements.txt
 
 Then run the scripts from this directory with:
 ```shell
-python3 build-sequencing-data.py
-python3 build-assemblies.py
-python3 build-annotations.py
+python3 build_sequencing-data.py
+python3 build_assemblies.py
+python3 build_annotations.py
 ```
 
 

catalog-build/build-catalog.ts

@@ -73,6 +73,7 @@ async function buildRawSequencingData(): Promise<
       deepConsensusVersion: row.DeepConsensus_version,
       designDescription: row.design_description,
       familyId: parseStringOrNull(row.familyID),
+      fileSize: parseNumberOrNA(row.file_size).toString(),
       filename: row.filename,
       filetype: row.filetype,
       fiveHundredkbPlus: parseNumberOrNA(row["500kb+"]).toString(),
@@ -139,6 +140,8 @@ async function buildAssemblies(): Promise<HPRCDataExplorerAssembly[]> {
       awsFasta: parseStringOrNull(row.aws_fasta),
       familyId: parseStringOrNull(row.familyID),
       fastaSha256: parseStringOrNull(row.fasta_sha256),
+      fileSize: parseNumberOrNA(row.file_size).toString(),
+      filename: getFileNameFromPath(row.aws_fasta),
       frag: parseNumberOrNull(row.frag),
       fullDup: parseNumberOrNull(row.full_dup),
       fullSgl: parseNumberOrNull(row.full_sgl),
@@ -170,6 +173,8 @@ async function buildAnnotations(): Promise<HPRCDataExplorerAnnotation[]> {
     (row): HPRCDataExplorerAnnotation => ({
       annotationType: row.annotation_type,
       fileLocation: row.file_location,
+      fileSize: parseNumberOrNA(row.file_size).toString(),
+      filename: getFileNameFromPath(row.file_location),
       haplotype: parseStringOrNull(row.haplotype),
       reference: parseStringOrNull(row.reference),
       sampleId: row.sample,
@@ -215,6 +220,10 @@ async function saveJson(filePath: string, data: unknown): Promise<void> {
   await fsp.writeFile(filePath, JSON.stringify(data, undefined, 2));
 }
 
+function getFileNameFromPath(p: string): string {
+  return p.substring(p.lastIndexOf("/") + 1);
+}
+
 function parseStringOrNull(value: string): string | null {
   return value.trim() || null;
 }