Merge pull request #53 from Insight-Services-APAC/feature/pip_install…

…able

Making project pip installable

MANIFEST.in

@@ -1 +1 @@
-recursive-include dbt/include *.sql *.yml *.md
+recursive-include dbt/include *.sql *.yml *.md *.ipynb *.ps1

dbt/adapters/fabricsparknb/notebook.py

@@ -4,16 +4,34 @@
 import io 
 import nbformat as nbf
 from jinja2 import Environment, FileSystemLoader
+from sysconfig import get_paths
+from pathlib import Path
+from sysconfig import get_paths
 
 
+@staticmethod
+def GetIncludeDir():
+    ChkPath = Path(get_paths()['purelib']) / Path(f'dbt/include/fabricsparknb/')
+    # print(ChkPath)
+    # Does Check for the path
+    if os.path.exists(ChkPath):
+        return ChkPath
+    else:        
+        path = Path(os.getcwd()) / Path('dbt/include/fabricsparknb/')
+        # print(str(path))
+        return (path)
+
 class ModelNotebook:
     def __init__(self, nb : nbf.NotebookNode = None, node_type='model'):
+
         if nb is None:
-            filename = f'dbt/include/fabricsparknb/notebooks/{node_type}_notebook.ipynb'
+            filename = str((GetIncludeDir()) / Path(f'notebooks/{node_type}_notebook.ipynb'))            
             if os.path.exists(filename):
                 with io.open(file=filename, mode='r', encoding='utf-8') as f:
                     file_str = f.read()
                     nb = nbf.reads(file_str, as_version=4)
+            else: 
+                raise Exception(f"Notebook file {filename} does not exist")
 
         self.nb: nbf.NotebookNode = nb
         self.sql: list[str] = []
@@ -55,16 +73,18 @@ def GetSparkSqlCells(self):
 
     def Render(self):        
         # Define the directory containing the Jinja templates
-        template_dir = 'dbt/include/fabricsparknb/notebooks/'
-
-        # Create a Jinja environment
-        env = Environment(loader=FileSystemLoader(template_dir))
+        template_dir = str((GetIncludeDir()) / Path('notebooks/'))
+        if os.path.exists(template_dir):
+            # Create a Jinja environment
+            env = Environment(loader=FileSystemLoader(template_dir))
 
-        # Load the template
-        template = env.get_template('model_notebook.ipynb')
+            # Load the template
+            template = env.get_template('model_notebook.ipynb')
 
-        # Render the template with the notebook_file variable
-        rendered_template = template.render()
+            # Render the template with the notebook_file variable
+            rendered_template = template.render()
 
-        # Parse the rendered template as a notebook
-        self.nb = nbf.reads(rendered_template, as_version=4)
+            # Parse the rendered template as a notebook
+            self.nb = nbf.reads(rendered_template, as_version=4)
+        else:
+            raise Exception(f"Directory {template_dir} does not exist")

dbt/adapters/fabricsparknb/utils.py

@@ -10,6 +10,7 @@
 import dbt.adapters.fabricsparknb.catalog as Catalog
 from dbt.clients.system import load_file_contents
 from dbt.adapters.fabricsparknb.notebook import ModelNotebook
+import dbt.adapters.fabricsparknb.notebook as mn
 from pathlib import Path
 #from azure.storage.filedatalake import (
 #    DataLakeServiceClient,
@@ -39,6 +40,7 @@ def CheckSqlForModelCommentBlock(sql) -> bool:
         return False
 
 
+
 @staticmethod
 def GenerateMasterNotebook(project_root, workspaceid, lakehouseid, lakehouse_name):
     # Iterate through the notebooks directory and create a list of notebook files
@@ -70,7 +72,7 @@ def GenerateMasterNotebook(project_root, workspaceid, lakehouseid, lakehouse_nam
         # Do something with the files...
 
         # Define the directory containing the Jinja templates
-        template_dir = 'dbt/include/fabricsparknb/notebooks/'
+        template_dir = str((mn.GetIncludeDir()) / Path('notebooks/'))
 
         # Create a Jinja environment
         env = Environment(loader=FileSystemLoader(template_dir))
@@ -97,7 +99,7 @@ def GenerateMasterNotebook(project_root, workspaceid, lakehouseid, lakehouse_nam
 
 
     # Define the directory containing the Jinja templates
-    template_dir = 'dbt/include/fabricsparknb/notebooks/'
+    template_dir = str((mn.GetIncludeDir()) / Path('notebooks/'))
 
     # Create a Jinja environment
     env = Environment(loader=FileSystemLoader(template_dir))
@@ -145,7 +147,7 @@ def GenerateMasterNotebook(project_root, workspaceid, lakehouseid, lakehouse_nam
 def GenerateMetadataExtract(project_root, workspaceid, lakehouseid, lakehouse_name):
     notebook_dir = f'./{project_root}/target/notebooks/'
     # Define the directory containing the Jinja templates
-    template_dir = 'dbt/include/fabricsparknb/notebooks/'
+    template_dir = str((mn.GetIncludeDir()) / Path('notebooks/'))
 
     # Create a Jinja environment
     env = Environment(loader=FileSystemLoader(template_dir))
@@ -174,7 +176,7 @@ def GenerateMetadataExtract(project_root, workspaceid, lakehouseid, lakehouse_na
 def GenerateNotebookUpload(project_root, workspaceid, lakehouseid, lakehouse_name):
     notebook_dir = f'./{project_root}/target/notebooks/'
     # Define the directory containing the Jinja templates
-    template_dir = 'dbt/include/fabricsparknb/notebooks/'
+    template_dir = str((mn.GetIncludeDir()) / Path('notebooks/'))
 
     # Create a Jinja environment
     env = Environment(loader=FileSystemLoader(template_dir))
@@ -205,7 +207,7 @@ def GenerateAzCopyScripts(project_root, workspaceid, lakehouseid):
 
     Path(notebook_dir).mkdir(parents=True, exist_ok=True)
     # Define the directory containing the Jinja templates
-    template_dir = 'dbt/include/fabricsparknb/pwsh/'
+    template_dir = str((mn.GetIncludeDir()) / Path('pwsh/'))    
 
     # Create a Jinja environment
     env = Environment(loader=FileSystemLoader(template_dir))