Merge pull request #793 from spectacles-ci/feature/use-personal-branch

Use personal branches instead of temp branches when specified

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spectacles"
-version = "2.4.7"
+version = "2.4.8"
 description = "A command-line, continuous integration tool for Looker and LookML."
 authors = ["Spectacles <[email protected]>"]
 license = "MIT"

spectacles/cli.py

@@ -331,6 +331,7 @@ def main() -> None:
                 chunk_size=args.chunk_size,
                 pin_imports=pin_imports,
                 ignore_hidden=args.ignore_hidden,
+                use_personal_branch=args.use_personal_branch,
             )
         )
     elif args.command == "assert":
@@ -346,6 +347,7 @@ def main() -> None:
                 api_version=args.api_version,
                 remote_reset=args.remote_reset,
                 pin_imports=pin_imports,
+                use_personal_branch=args.use_personal_branch,
                 concurrency=args.concurrency,
             )
         )
@@ -366,6 +368,7 @@ def main() -> None:
                 exclude_personal=args.exclude_personal,
                 folders=[restore_dash(arg) for arg in args.folders],
                 pin_imports=pin_imports,
+                use_personal_branch=args.use_personal_branch,
             )
         )
     elif args.command == "lookml":
@@ -381,6 +384,7 @@ def main() -> None:
                 remote_reset=args.remote_reset,
                 severity=args.severity,
                 pin_imports=pin_imports,
+                use_personal_branch=args.use_personal_branch,
                 timeout=args.timeout,
             )
         )
@@ -536,6 +540,19 @@ def _build_validator_subparser(
         env_var="LOOKER_GIT_BRANCH",
         help="The branch of your project that Spectacles will use to run queries.",
     )
+    base_subparser.add_argument(
+        "--use-personal-branch",
+        action=EnvVarStoreTrueAction,
+        env_var="SPECTACLES_USE_PERSONAL_BRANCH",
+        help="Use the user's personal branch instead of creating a temporary branch for the tests.",
+    )
+    base_subparser.add_argument(
+        "--pin-imports",
+        nargs="+",
+        default=[],
+        help="Pin locally imported Looker projects to a specific ref (Git branch or commit) during validation. \
+            Provide these arguments in project_name:ref format.",
+    )
     group = base_subparser.add_mutually_exclusive_group()
     group.add_argument(
         "--remote-reset",
@@ -553,13 +570,6 @@ def _build_validator_subparser(
             In order to test a specific commit, Spectacles will create a new branch \
             for the tests and then delete the branch when it is finished.",
     )
-    group.add_argument(
-        "--pin-imports",
-        nargs="+",
-        default=[],
-        help="Pin locally imported Looker projects to a specific ref (Git branch or commit) during validation. \
-            Provide these arguments in project_name:ref format.",
-    )
     return base_subparser
 
 
@@ -802,15 +812,16 @@ async def run_lookml(
     remote_reset: bool,
     severity: str,
     pin_imports: Dict[str, str],
-    timeout: int = LOOKML_VALIDATION_TIMEOUT,
+    use_personal_branch: bool,
+    timeout: int,
 ) -> None:
     # Don't trust env to ignore .netrc credentials
     async_client = httpx.AsyncClient(trust_env=False)
     try:
         client = LookerClient(
             async_client, base_url, client_id, client_secret, port, api_version
         )
-        runner = Runner(client, project, remote_reset, pin_imports)
+        runner = Runner(client, project, remote_reset, pin_imports, use_personal_branch)
 
         results = await runner.validate_lookml(ref, severity, timeout)
     finally:
@@ -861,14 +872,15 @@ async def run_content(
     exclude_personal: bool,
     folders: List[str],
     pin_imports: Dict[str, str],
+    use_personal_branch: bool,
 ) -> None:
     # Don't trust env to ignore .netrc credentials
     async_client = httpx.AsyncClient(trust_env=False)
     try:
         client = LookerClient(
             async_client, base_url, client_id, client_secret, port, api_version
         )
-        runner = Runner(client, project, remote_reset, pin_imports)
+        runner = Runner(client, project, remote_reset, pin_imports, use_personal_branch)
 
         results = await runner.validate_content(
             ref,
@@ -920,6 +932,7 @@ async def run_assert(
     api_version: float,
     remote_reset: bool,
     pin_imports: Dict[str, str],
+    use_personal_branch: bool,
     concurrency: int,
 ) -> None:
     # Don't trust env to ignore .netrc credentials
@@ -928,7 +941,7 @@ async def run_assert(
         client = LookerClient(
             async_client, base_url, client_id, client_secret, port, api_version
         )
-        runner = Runner(client, project, remote_reset, pin_imports)
+        runner = Runner(client, project, remote_reset, pin_imports, use_personal_branch)
 
         results = await runner.validate_data_tests(ref, filters, concurrency)
     finally:
@@ -981,6 +994,7 @@ async def run_sql(
     runtime_threshold: int,
     chunk_size: int,
     pin_imports: Dict[str, str],
+    use_personal_branch: bool,
     ignore_hidden: bool,
 ) -> None:
     """Runs and validates the SQL for each selected LookML dimension."""
@@ -990,7 +1004,7 @@ async def run_sql(
         client = LookerClient(
             async_client, base_url, client_id, client_secret, port, api_version
         )
-        runner = Runner(client, project, remote_reset, pin_imports)
+        runner = Runner(client, project, remote_reset, pin_imports, use_personal_branch)
 
         results = await runner.validate_sql(
             ref,

spectacles/client.py

@@ -258,7 +258,7 @@ async def update_workspace(self, workspace: str) -> None:
         self.workspace = workspace
 
     @backoff.on_exception(backoff.expo, BACKOFF_EXCEPTIONS, max_tries=DEFAULT_MAX_TRIES)
-    async def get_all_branches(self, project: str) -> List[str]:
+    async def get_all_branches(self, project: str) -> List[JsonDict]:
         """Returns a list of git branches in the project repository.
 
         Args:
@@ -283,7 +283,7 @@ async def get_all_branches(self, project: str) -> List[str]:
                 response=response,
             ) from error
 
-        return [branch["name"] for branch in response.json()]
+        return response.json()  # type: ignore[no-any-return]
 
     @backoff.on_exception(backoff.expo, BACKOFF_EXCEPTIONS, max_tries=DEFAULT_MAX_TRIES)
     async def checkout_branch(self, project: str, branch: str) -> None:

spectacles/runner.py

@@ -44,6 +44,7 @@ def __init__(
         client: LookerClient,
         project: str,
         remote_reset: bool = False,
+        use_personal_branch: bool = False,
         pin_imports: Optional[Dict[str, str]] = None,
         skip_imports: Optional[List[str]] = None,
     ):
@@ -58,6 +59,8 @@ def __init__(
         self.commit: Optional[str] = None
         self.branch: Optional[str] = None
         self.is_temp_branch: bool = False
+        self.use_personal_branch: bool = use_personal_branch
+        self.personal_branch: Optional[str] = None
         self.import_managers: List[LookerBranchManager] = []
         self.skip_imports: List[str] = [] if skip_imports is None else skip_imports
 
@@ -108,15 +111,19 @@ async def __aenter__(self) -> None:
         if self.branch:
             await self.update_workspace("dev")
             if self.ephemeral:
-                self.branch = await self.checkout_temp_branch("origin/" + self.branch)
+                new_branch = await self.checkout_ephemeral_branch(
+                    "origin/" + self.branch
+                )
+                if not self.use_personal_branch:
+                    self.branch = new_branch
             else:
                 await self.client.checkout_branch(self.project, self.branch)
                 if self.remote_reset:
                     await self.client.reset_to_remote(self.project)
         # A commit was passed, so we non-destructively create a temporary branch we can
         # hard reset to the commit.
         elif self.commit:
-            self.branch = await self.checkout_temp_branch(self.commit)
+            self.branch = await self.checkout_ephemeral_branch(self.commit)
         # Neither branch nor commit were passed, so go to production.
         else:
             if self.init_state.workspace == "production":
@@ -127,7 +134,7 @@ async def __aenter__(self) -> None:
             self.branch = prod_state.branch
             self.commit = prod_state.commit
             if self.ephemeral:
-                self.branch = await self.checkout_temp_branch(prod_state.commit)
+                self.branch = await self.checkout_ephemeral_branch(prod_state.commit)
 
         logger.debug(
             f"Set project '{self.project}' to branch '{self.branch}' @ "
@@ -163,6 +170,7 @@ async def __aenter__(self) -> None:
                         project,
                         pin_imports=self.pin_imports,
                         skip_imports=self.skip_imports,
+                        use_personal_branch=self.use_personal_branch,
                     )
                     await manager(ref=import_ref, ephemeral=True).__aenter__()
                     self.import_managers.append(manager)
@@ -250,6 +258,27 @@ async def get_project_imports(self) -> List[str]:
         else:
             return [p["name"] for p in manifest["imports"] if not p["is_remote"]]
 
+    async def checkout_personal_branch(self, ref: str) -> str:
+        """Updates the user's personal branch to the git ref."""
+        await self.update_workspace("dev")
+        if not self.personal_branch:
+            self.personal_branch = await self.get_personal_branch()
+        await self.client.checkout_branch(self.project, self.personal_branch)
+        await self.client.reset_to_remote(self.project)
+        await self.client.hard_reset_branch(self.project, self.personal_branch, ref)
+        return self.personal_branch
+
+    async def get_personal_branch(self) -> str:
+        """Finds the name of the user's personal branch."""
+        branches = await self.client.get_all_branches(self.project)
+        for branch in branches:
+            if branch["personal"] and not branch["readonly"]:
+                return str(branch["name"])
+        raise ValueError(
+            f"Personal branch not found for client ID {self.client.client_id} "
+            f"in project '{self.project}'"
+        )
+
     async def checkout_temp_branch(self, ref: str) -> str:
         """Creates a temporary branch off a commit or off production."""
         # Save the dev mode state so we have somewhere to delete the temp branch
@@ -267,6 +296,14 @@ async def checkout_temp_branch(self, ref: str) -> str:
         self.is_temp_branch = True
         return name
 
+    async def checkout_ephemeral_branch(self, ref: str) -> str:
+        """Either check out temp or personal branch and hard-reset."""
+        if self.use_personal_branch:
+            branch = await self.checkout_personal_branch(ref)
+        else:
+            branch = await self.checkout_temp_branch(ref)
+        return branch
+
 
 class Runner:
     """Runs validations and returns JSON-style dictionaries with validation results.
@@ -291,11 +328,16 @@ def __init__(
         project: str,
         remote_reset: bool = False,
         pin_imports: Optional[Dict[str, str]] = None,
+        use_personal_branch: bool = False,
     ):
         self.project = project
         self.client = client
         self.branch_manager = LookerBranchManager(
-            client, project, remote_reset, pin_imports or {}
+            client,
+            project,
+            remote_reset,
+            pin_imports=pin_imports or {},
+            use_personal_branch=use_personal_branch,
         )
 
     async def validate_sql(

tests/integration/test_branch_manager.py

@@ -145,7 +145,8 @@ async def test_manage_current_branch_with_ref(
     branch_info = await looker_client.get_active_branch(LOOKER_PROJECT)
     assert branch_info["name"] == starting_branch
     assert branch_info["ref"][:6] != commit
-    all_branches = await looker_client.get_all_branches(LOOKER_PROJECT)
+    all_branches_json = await looker_client.get_all_branches(LOOKER_PROJECT)
+    all_branches = [branch["name"] for branch in all_branches_json]
     assert temp_branch not in all_branches
 
 
@@ -211,7 +212,8 @@ async def test_manage_other_branch_with_import_projects(
     assert active_branch == starting_branch
     active_branch = await looker_client.get_active_branch_name(dependent_project)
     assert active_branch == dependent_project_manager.init_state.branch
-    all_branches = await looker_client.get_all_branches(dependent_project)
+    all_branches_json = await looker_client.get_all_branches(dependent_project)
+    all_branches = [branch["name"] for branch in all_branches_json]
     assert temp_branch not in all_branches
 
 
@@ -269,7 +271,8 @@ async def test_manage_with_ref_import_projects(
     assert branch_info["ref"][:6] != commit
 
     await looker_client.update_workspace("dev")
-    all_branches = set(await looker_client.get_all_branches(dependent_project))
+    all_branches_json = await looker_client.get_all_branches(dependent_project)
+    all_branches = {branch["name"] for branch in all_branches_json}
     # Confirm that no temp branches still remain
     temp_branches = set(
         import_manager.branch for import_manager in manager.import_managers