Crucial fixes for xfstests and kernel_source_packager code

lisa/microsoft/testsuites/xfstests/xfstesting.py

@@ -1355,9 +1355,6 @@ def verify_azure_file_share(
                 run_id_prefix="cifs_worker",
             )
 
-            # Send deferred notifications now that parallel execution is complete
-            runner.send_deferred_notifications(worker_results, result)
-
             # Aggregate results (raises on failure)
             _, _, ctx.test_failed = runner.aggregate_results(worker_results)
 
@@ -1478,9 +1475,6 @@ def verify_azure_file_share_nfsv4(
                 run_id_prefix="nfs_worker",
             )
 
-            # Send deferred notifications now that parallel execution is complete
-            runner.send_deferred_notifications(worker_results, result)
-
             # Aggregate results (raises on failure)
             _, _, ctx.test_failed = runner.aggregate_results(worker_results)
 

lisa/sut_orchestrator/azure/common.py

@@ -29,7 +29,7 @@
 import requests
 from assertpy import assert_that
 from azure.core.credentials import AccessToken, TokenCredential
-from azure.core.exceptions import ResourceExistsError
+from azure.core.exceptions import HttpResponseError, ResourceExistsError
 from azure.keyvault.certificates import (
     CertificateClient,
     CertificatePolicy,
@@ -2388,7 +2388,34 @@ def get_or_create_file_share(
                 log.debug(
                     f"  provisioned_bandwidth_mibps: {provisioned_bandwidth_mibps}"
                 )
-            share_service_client.create_share(file_share_name, **create_kwargs)
+
+            try:
+                share_service_client.create_share(file_share_name, **create_kwargs)
+            except HttpResponseError as e:
+                # Handle PV2 parameters unsupported by the storage account
+                if "UnsupportedHeader" in str(e) and (
+                    "x-ms-share-provisioned-iops" in str(e)
+                    or "x-ms-share-provisioned-bandwidth" in str(e)
+                ):
+                    log.info(
+                        "PV2 parameters not supported by storage account, "
+                        "retrying without provisioned_iops/provisioned_bandwidth_mibps"
+                    )
+
+                    # Remove PV2-specific parameters and retry
+                    create_kwargs.pop("provisioned_iops", None)
+                    create_kwargs.pop("provisioned_bandwidth_mibps", None)
+
+                    # For non-PV2 premium shares, minimum quota is 100 GiB
+                    if create_kwargs.get("quota", 0) < 100:
+                        log.info(
+                            f"Increasing quota from {create_kwargs.get('quota')} "
+                            "to 100 GiB (minimum for non-PV2 premium file shares)"
+                        )
+                        create_kwargs["quota"] = 100
+                    share_service_client.create_share(file_share_name, **create_kwargs)
+                else:
+                    raise
         return str("//" + share_service_client.primary_hostname + "/" + file_share_name)
 
 

lisa/tools/git.py

@@ -614,7 +614,7 @@ def worktree_list(
     def worktree_remove(
         self,
         cwd: pathlib.PurePath,
-        path: pathlib.PurePath,
+        path: str,
         force: bool = False,
     ) -> None:
         cmd = "worktree remove"

lisa/transformers/kernel_source_installer.py

@@ -462,12 +462,12 @@ def _build_code(
 
         make_args = ""
         if use_ccache:
-            make_args = "CC='ccache gcc'"
-            node.execute(
-                cmd=f"export CCACHE_DIR={str(code_path.parent)}/.ccache",
-                shell=True,
-                no_error_log=True,
-            )
+            ccache_dir = code_path.parent / ".ccache"
+            if not node.shell.exists(ccache_dir):
+                node.execute(f"mkdir -p {ccache_dir}", sudo=True)
+                node.execute(f"chmod 0777 {ccache_dir}", sudo=True)
+
+            make_args = f"CC='ccache gcc' CCACHE_DIR={str(ccache_dir)}"
 
         # set timeout to 2 hours
         make.make(arguments=make_args, cwd=code_path, timeout=60 * 60 * 2)

lisa/transformers/kernel_source_packager.py

@@ -32,7 +32,6 @@ class RepoWorktreeSchema(RepoLocationSchema):
     worktree_name: str = ""
     worktree_repo: str = ""
     worktree_ref: str = ""
-    worktree_local_branch: str = ""
 
 
 @dataclass_json()
@@ -388,6 +387,131 @@ def type_name(cls) -> str:
     def type_schema(cls) -> Type[schema.TypedSchema]:
         return RepoWorktreeSchema
 
+    def _cleanup_detached_worktrees(
+        self,
+        code_path: PurePath,
+    ) -> None:
+        git = self._node.tools[Git]
+
+        # Remove detached worktrees and their associated branches.
+        worktrees = git.worktree_list(cwd=code_path)
+        detached_worktrees = [
+            wt["path"] for wt in worktrees if wt.get("branch") == "(detached)"
+        ]
+
+        for worktree in detached_worktrees:
+            try:
+                git.worktree_remove(cwd=code_path, path=worktree)
+                # Also deleting branches created by the worktree for coherence.
+                # This doesn't work for mainline in
+                # /mnt/code/linux pointed by origin.
+                # Use basename since worktree is a full path from worktree_list.
+                worktree_name = PurePath(worktree).name
+                self._node.execute(
+                    f"git branch --list '{worktree_name}-*' | xargs git branch -D",
+                    shell=True,
+                    no_error_log=True,
+                    cwd=code_path,
+                )
+            except Exception as e:
+                self._log.debug(
+                    "failed to cleanup detached worktree and branches "
+                    f"'{worktree}': {e}"
+                )
+
+        git.worktree_prune(cwd=code_path)
+
+    def _is_tag(
+        self,
+        target_path: PurePath,
+        ref: str,
+    ) -> bool:
+        result = self._node.execute(
+            f"git tag -l {ref}",
+            shell=True,
+            cwd=target_path,
+        )
+        return bool(result.stdout.strip())
+
+    def _checkout_target_ref(
+        self,
+        target_path: PurePath,
+        target_ref: str,
+        remote: str,
+    ) -> None:
+        git = self._node.tools[Git]
+
+        # Tags are not namespaced under remotes (there is no remote/tag-name).
+        # Detect tags and use the ref directly instead of remote/ref.
+        is_tag = self._is_tag(target_path, target_ref)
+        if is_tag:
+            remote_ref = target_ref
+            self._log.debug(
+                f"'{target_ref}' is a tag, using it directly instead of "
+                f"'{remote}/{target_ref}'."
+            )
+        else:
+            remote_ref = f"{remote}/{target_ref}"
+
+        # Checkout and update the target ref, with force-checkout fallback.
+        expected_local_branch_name = f"{remote}-{target_ref}"
+        try:
+            local_branches_result = self._node.execute(
+                "git branch --format='%(refname:short)'",
+                shell=True,
+                cwd=target_path,
+            )
+            local_branches = [
+                b.strip().strip("'")
+                for b in local_branches_result.stdout.splitlines()
+                if b.strip()
+            ]
+            if expected_local_branch_name in local_branches:
+                self._log.debug("Pulling upstream in an existing branch.")
+                self._node.execute(
+                    f"git checkout -f {expected_local_branch_name}",
+                    shell=True,
+                    cwd=target_path,
+                    expected_exit_code=0,
+                )
+                if not is_tag:
+                    git.pull(cwd=target_path)
+            elif target_ref in local_branches:
+                self._log.debug("Pulling upstream in an existing branch.")
+                self._node.execute(
+                    f"git checkout -f {target_ref}",
+                    shell=True,
+                    cwd=target_path,
+                    expected_exit_code=0,
+                )
+                if not is_tag:
+                    git.pull(cwd=target_path)
+            else:
+                self._log.debug("Checking out a new branch synced with remote.")
+                self._node.execute(
+                    f"git checkout -b {expected_local_branch_name} {remote_ref}",
+                    shell=True,
+                    cwd=target_path,
+                    expected_exit_code=0,
+                )
+
+            self._log.info(
+                f"checkout code from: '{target_ref}', in "
+                f"'{git.get_current_branch(cwd=target_path)}'"
+            )
+        except Exception:
+            self._log.debug("Checking out a new branch force synced with remote")
+            self._node.execute(
+                f"git checkout -B {expected_local_branch_name} {remote_ref}",
+                shell=True,
+                cwd=target_path,
+                expected_exit_code=0,
+            )
+            self._log.info(
+                f"checkout code from: '{target_ref}', in "
+                f"'{git.get_current_branch(cwd=target_path)}'"
+            )
+
     def get_source_code(self) -> PurePath:
         runbook: RepoWorktreeSchema = cast(RepoWorktreeSchema, self.runbook)
 
@@ -418,31 +542,42 @@ def get_source_code(self) -> PurePath:
         else:
             code_path = code_path / repo_name
 
-        # check if the 'repo' is already a remote url
-        remote_exists = False
-        remote = ""
+        remote = "origin"
         remotes = git.remote_list(code_path)
         self._log.debug(f"existing remotes: {remotes}")
-        for remote in remotes:
-            if runbook.worktree_repo == git.remote_get_url(code_path, remote):
-                remote_exists = True
-                break
-
-        if not remote_exists:
-            remote = runbook.worktree_name
-            self._log.info(f"adding remote {remote} for {runbook.worktree_repo}")
-            git.remote_add(cwd=code_path, name=remote, url=runbook.worktree_repo)
-        git.fetch(
+        if runbook.worktree_name:
+            if runbook.worktree_name in remotes:
+                self._log.debug("Setting the remote based on worktree name/path.")
+                remote = runbook.worktree_name
+                if runbook.worktree_repo:
+                    assert runbook.worktree_repo == git.remote_get_url(
+                        code_path, remote
+                    ), f"Existing remote url doesn't match with {runbook.worktree_repo}"
+            elif runbook.worktree_name == repo_name:
+                self._log.debug("Using the upstream repo pointed by 'origin' remote")
+
+            else:
+                assert runbook.worktree_repo, "Remote can not be added without a URL"
+                remote = runbook.worktree_name
+                self._log.info(f"adding remote {remote} for {runbook.worktree_repo}")
+                git.remote_add(cwd=code_path, name=remote, url=runbook.worktree_repo)
+
+        self._node.execute(
+            f"git fetch -p {remote} --force --tags",
+            shell=True,
+            no_info_log=False,
             cwd=code_path,
-            remote=remote,
+            expected_exit_code=0,
         )
 
         target_path = code_path
         target_ref = runbook.ref
         if runbook.worktree_name:
+            self._cleanup_detached_worktrees(code_path)
+
             worktree_path = code_path.parent / runbook.worktree_name
-            git.worktree_prune(cwd=code_path)
             if not git.worktree_exists(cwd=code_path, path=str(worktree_path)):
+                assert runbook.worktree_ref, "Worktree ref needs to be set by user"
                 self._log.info(
                     f"creating a new worktree at {worktree_path} "
                     f"pointing at {remote}/{runbook.worktree_ref}"
@@ -452,24 +587,22 @@ def get_source_code(self) -> PurePath:
                     path=worktree_path,
                     remote=remote,
                     remote_ref=runbook.worktree_ref,
-                    new_branch=runbook.worktree_local_branch,
+                    new_branch=f"{remote}-{runbook.worktree_ref}",
                     track=True,
                 )
 
                 latest_commit_id = git.get_latest_commit_id(cwd=worktree_path)
                 self._log.info(f"Kernel HEAD is now at : {latest_commit_id}")
                 return worktree_path
 
-            # worktree exists
-            target_ref = runbook.worktree_ref
+            # worktree exists — fetch the tracking remote and update
+            self._log.debug("Using the existing worktree")
+            if runbook.worktree_ref:
+                target_ref = runbook.worktree_ref
             target_path = worktree_path
 
         if target_ref:
-            if git.get_current_branch(cwd=target_path) == target_ref:
-                git.pull(cwd=target_path)
-
-            git.checkout(ref=target_ref, cwd=target_path)
-            self._log.info(f"checkout code from: '{target_ref}'")
+            self._checkout_target_ref(target_path, target_ref, remote)
 
         latest_commit_id = git.get_latest_commit_id(cwd=target_path)
         self._log.info(f"Kernel HEAD is now at : {latest_commit_id}")

pyproject.toml

@@ -53,7 +53,7 @@ azure = [
     "azure-mgmt-serialconsole ~= 1.0.0",
     "azure-mgmt-storage ~= 21.2.1",
     "azure-storage-blob ~= 12.23.0",
-    "azure-storage-file-share ~= 12.20.0",
+    "azure-storage-file-share ~= 12.24.0",
     "azure-keyvault-secrets ~= 4.7.0",
     "azure-keyvault-certificates ~= 4.7.0",
     "msrestazure ~= 0.6.4",