alembic sync (#1336)

### Feature or Bugfix - Bugfix ### Detail - Enum `DatasetType` renamed to `DatasetTypes`. In previous migrations name of the type `datasettypes` is used, so alembic detected it as a change, because automatically generated `DatasetType` -> `datasettype`. - `dataset_lock` is now nullable - Omics models are imported into migrations/env.py - Omics fileds are updated ### Relates - ### Security Please answer the questions below briefly where applicable, or write `N/A`. Based on [OWASP 10](https://owasp.org/Top10/en/). - Does this PR introduce or modify any input fields or queries - this includes fetching data from storage outside the application (e.g. a database, an S3 bucket)? - Is the input sanitized? - What precautions are you taking before deserializing the data you consume? - Is injection prevented by parametrizing queries? - Have you ensured no `eval` or similar functions are used? - Does this PR introduce any functionality or component that requires authorization? - How have you ensured it respects the existing AuthN/AuthZ mechanisms? - Are you logging failed auth attempts? - Are you using or adding any cryptographic features? - Do you use a standard proven implementations? - Are the used keys controlled by the customer? Where are they stored? - Are you introducing any new policies/roles/users? - Have you used the least-privilege principle? How? By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --------- Co-authored-by: Sofia Sazonova <[email protected]>
data-dot-all · Jun 19, 2024 · 94b71c5 · 94b71c5
1 parent d3f98da
commit 94b71c5
Show file tree

Hide file tree

Showing 10 changed files with 100 additions and 20 deletions.
diff --git a/backend/dataall/modules/datasets_base/db/dataset_models.py b/backend/dataall/modules/datasets_base/db/dataset_models.py
@@ -2,7 +2,7 @@
 from sqlalchemy.dialects.postgresql import JSON, ARRAY
 from sqlalchemy.orm import query_expression
 from dataall.base.db import Base, Resource, utils
-from dataall.modules.datasets_base.services.datasets_enums import ConfidentialityClassification, Language, DatasetType
+from dataall.modules.datasets_base.services.datasets_enums import ConfidentialityClassification, Language, DatasetTypes
 
 
 class DatasetBase(Resource, Base):
@@ -29,7 +29,7 @@ class DatasetBase(Resource, Base):
     SamlAdminGroupName = Column(String, nullable=True)
     autoApprovalEnabled = Column(Boolean, default=False)
 
-    datasetType = Column(Enum(DatasetType), nullable=False, default=DatasetType.S3)
+    datasetType = Column(Enum(DatasetTypes), nullable=False, default=DatasetTypes.S3)
     imported = Column(Boolean, default=False)
 
     __mapper_args__ = {'polymorphic_identity': 'dataset', 'polymorphic_on': datasetType}

diff --git a/backend/dataall/modules/datasets_base/services/dataset_service_interface.py b/backend/dataall/modules/datasets_base/services/dataset_service_interface.py
@@ -1,6 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
-from dataall.modules.datasets_base.services.datasets_enums import DatasetType
+from dataall.modules.datasets_base.services.datasets_enums import DatasetTypes
 
 log = logging.getLogger(__name__)
 
@@ -13,7 +13,7 @@ class DatasetServiceInterface(ABC):
 
     @property
     @abstractmethod
-    def dataset_type(self) -> DatasetType: ...
+    def dataset_type(self) -> DatasetTypes: ...
 
     @staticmethod
     @abstractmethod

diff --git a/backend/dataall/modules/datasets_base/services/datasets_enums.py b/backend/dataall/modules/datasets_base/services/datasets_enums.py
@@ -5,7 +5,7 @@
 custom_confidentiality_mapping = config.get_property('modules.s3_datasets.features.custom_confidentiality_mapping', {})
 
 
-class DatasetType(GraphQLEnumMapper):
+class DatasetTypes(GraphQLEnumMapper):
     S3 = 'S3'
 
 

diff --git a/backend/dataall/modules/s3_datasets/db/dataset_models.py b/backend/dataall/modules/s3_datasets/db/dataset_models.py
@@ -3,7 +3,7 @@
 from sqlalchemy.orm import query_expression
 from dataall.base.db import Base, Resource, utils
 from dataall.modules.datasets_base.db.dataset_models import DatasetBase
-from dataall.modules.datasets_base.services.datasets_enums import DatasetType
+from dataall.modules.datasets_base.services.datasets_enums import DatasetTypes
 
 
 class DatasetTableColumn(Resource, Base):
@@ -114,7 +114,7 @@ class S3Dataset(DatasetBase):
     importedAdminRole = Column(Boolean, default=False)
 
     __mapper_args__ = {
-        'polymorphic_identity': DatasetType.S3,
+        'polymorphic_identity': DatasetTypes.S3,
     }
 
 

diff --git a/backend/dataall/modules/s3_datasets_shares/services/dataset_sharing_service.py b/backend/dataall/modules/s3_datasets_shares/services/dataset_sharing_service.py
@@ -21,7 +21,7 @@
     CREDENTIALS_DATASET,
 )
 from dataall.modules.s3_datasets.db.dataset_models import S3Dataset
-from dataall.modules.datasets_base.services.datasets_enums import DatasetRole, DatasetType
+from dataall.modules.datasets_base.services.datasets_enums import DatasetRole, DatasetTypes
 from dataall.modules.datasets_base.services.dataset_service_interface import DatasetServiceInterface
 from dataall.modules.s3_datasets_shares.aws.glue_client import GlueClient
 
@@ -34,7 +34,7 @@
 class DatasetSharingService(DatasetServiceInterface):
     @property
     def dataset_type(self):
-        return DatasetType.S3
+        return DatasetTypes.S3
 
     @staticmethod
     def resolve_additional_dataset_user_role(session, uri, username, groups):

diff --git a/backend/migrations/env.py b/backend/migrations/env.py
@@ -17,6 +17,7 @@
 from dataall.modules.notifications.db.notification_models import Notification
 from dataall.modules.vote.db.vote_models import Vote
 from dataall.modules.worksheets.db.worksheet_models import WorksheetQueryResult, Worksheet
+from dataall.modules.omics.db.omics_models import OmicsWorkflow, OmicsRun
 # fmt: on
 # enable ruff-format back
 

diff --git a/backend/migrations/versions/448d9dc95e94_dataset_lock_and_types_adjustment.py b/backend/migrations/versions/448d9dc95e94_dataset_lock_and_types_adjustment.py
@@ -0,0 +1,29 @@
+"""dataset_lock_and_types_adjustment
+
+Revision ID: 448d9dc95e94
+Revises: e52fa2b8c887
+Create Date: 2024-06-17 16:42:34.851166
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '448d9dc95e94'
+down_revision = 'e52fa2b8c887'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('dataset_lock', 'isLocked', existing_type=sa.BOOLEAN(), nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('dataset_lock', 'isLocked', existing_type=sa.BOOLEAN(), nullable=False)
+    # ### end Alembic commands ###
diff --git a/backend/migrations/versions/5cdcf6cc1d73_create_and_backfill_dataset_table.py b/backend/migrations/versions/5cdcf6cc1d73_create_and_backfill_dataset_table.py
@@ -16,7 +16,7 @@
 from dataall.base.db import utils, Resource
 from sqlalchemy.ext.declarative import declarative_base
 from dataall.modules.datasets_base.services.datasets_enums import ConfidentialityClassification, Language
-from dataall.modules.datasets_base.services.datasets_enums import DatasetType
+from dataall.modules.datasets_base.services.datasets_enums import DatasetTypes
 
 # revision identifiers, used by Alembic.
 revision = '5cdcf6cc1d73'
@@ -51,7 +51,9 @@ def upgrade():
         sa.Column('SamlAdminGroupName', sa.String(), nullable=True),
         sa.Column('autoApprovalEnabled', sa.Boolean(), default=False),
         sa.Column(
-            'datasetType', postgresql.ENUM(DatasetType.S3.value, name='datasettypes', create_type=False), nullable=False
+            'datasetType',
+            postgresql.ENUM(DatasetTypes.S3.value, name='datasettypes', create_type=False),
+            nullable=False,
         ),
         sa.Column('imported', sa.Boolean(), default=False),
         sa.ForeignKeyConstraint(
@@ -309,9 +311,9 @@ class S3Dataset(Resource, Base):  # Old S3 Dataset class with all columns
         's3_dataset',
         sa.Column(
             'datasetType',
-            sa.Enum(DatasetType.S3.value, name='datasettypes'),
+            sa.Enum(DatasetTypes.S3.value, name='datasettypes'),
             nullable=False,
-            server_default=DatasetType.S3.value,
+            server_default=DatasetTypes.S3.value,
         ),
     )
     op.add_column(

diff --git a/backend/migrations/versions/d059eead99c2_rename_dataset_table_as_s3_dataset.py b/backend/migrations/versions/d059eead99c2_rename_dataset_table_as_s3_dataset.py
@@ -8,7 +8,7 @@
 
 from alembic import op
 import sqlalchemy as sa
-from dataall.base.api.constants import GraphQLEnumMapper
+from dataall.modules.datasets_base.services.datasets_enums import DatasetTypes
 
 # revision identifiers, used by Alembic.
 revision = 'd059eead99c2'
@@ -17,10 +17,6 @@
 depends_on = None
 
 
-class DatasetType(GraphQLEnumMapper):
-    S3 = 'S3'
-
-
 def upgrade():
     print('Renaming dataset as s3_dataset...')
     op.drop_constraint(constraint_name='fk_dataset_env_uri', table_name='dataset', type_='foreignkey')
@@ -39,9 +35,9 @@ def upgrade():
         's3_dataset',
         sa.Column(
             'datasetType',
-            sa.Enum(DatasetType.S3.value, name='datasettypes'),
+            sa.Enum(DatasetTypes.S3.value, name='datasettypes'),
             nullable=False,
-            server_default=DatasetType.S3.value,
+            server_default=DatasetTypes.S3.value,
         ),
     )
 

diff --git a/backend/migrations/versions/e52fa2b8c887_omics_adjustments.py b/backend/migrations/versions/e52fa2b8c887_omics_adjustments.py
@@ -0,0 +1,52 @@
+"""omics_adjustments
+
+Revision ID: e52fa2b8c887
+Revises: f2f7431c34e5
+Create Date: 2024-06-17 14:45:06.997065
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'e52fa2b8c887'
+down_revision = 'f2f7431c34e5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('omics_run', 'outputUri', existing_type=sa.VARCHAR(), nullable=True)
+    op.alter_column('omics_run', 'outputDatasetUri', existing_type=sa.VARCHAR(), nullable=True)
+    op.create_foreign_key(
+        'omics_run_workflowUri_fkey',
+        'omics_run',
+        'omics_workflow',
+        ['workflowUri'],
+        ['workflowUri'],
+        ondelete='cascade',
+    )
+    op.create_foreign_key(
+        'omics_run_environmentUri_fkey',
+        'omics_run',
+        'environment',
+        ['environmentUri'],
+        ['environmentUri'],
+        ondelete='cascade',
+    )
+    op.alter_column('omics_workflow', 'environmentUri', existing_type=sa.VARCHAR(), nullable=True)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('omics_workflow', 'environmentUri', existing_type=sa.VARCHAR(), nullable=False)
+    op.drop_constraint('omics_run_workflowUri_fkey', 'omics_run', type_='foreignkey')
+    op.drop_constraint('omics_run_environmentUri_fkey', 'omics_run', type_='foreignkey')
+    op.alter_column('omics_run', 'outputDatasetUri', existing_type=sa.VARCHAR(), nullable=False)
+    op.alter_column('omics_run', 'outputUri', existing_type=sa.VARCHAR(), nullable=False)
+
+    # ### end Alembic commands ###