download specs for koji scratch builds

from their SRPMs - this reintroduces the codebase
originally written by @nikromen, hence the co-authorship

If fetching of a spec file fails, return None instead of erroring out.
We want to have the annotated snippets and can live if a few don't have
spec files.

Fixes #98

Co-authored-by: Jiri Kyjovsky <[email protected]>
Signed-off-by: Tomas Tomecek <[email protected]>

backend/fetcher.py

@@ -1,10 +1,12 @@
 import binascii
+import logging
 import os
 import re
 import subprocess
 from abc import ABC, abstractmethod
 from functools import cached_property
 from http import HTTPStatus
+from pathlib import Path
 from typing import Optional
 from urllib.error import HTTPError
 
@@ -16,6 +18,7 @@
 from backend.constants import COPR_RESULT_TEMPLATE
 from backend.data import LOG_OUTPUT
 from backend.exceptions import FetchError
+from backend.spells import get_temporary_dir
 
 
 def handle_errors(func):
@@ -215,14 +218,16 @@ def task_info(self) -> dict:
     def task_request(self) -> list:
         return self.client.getTaskRequest(self.task_id)
 
-    def get_task_request_url(self) -> str:
+    def get_task_request_url(self) -> Optional[str]:
         """
         We need this:
             'git+https://src.fedoraproject.org/rpms/libphonenumber.git#c88bd3...
         This info is in self.task_request[0]
+        methods build and buildFromSCM has this
+        buildArch contains file-based path to SRPM
 
-        But not every task has this though, buildArch
-        contains file-based path to SRPM, build and buildFromSCM has it
+        For scratch builds submitted from CLI:
+            'cli-build/1705395313.3717997.mjCDejui/sqlite-3.45.0-1.fc40.src.rpm'
         """
         task_request_url = self.task_request[0]
         if task_request_url.startswith("git+https"):
@@ -232,25 +237,17 @@ def get_task_request_url(self) -> str:
             task_request_url = self.client.getTaskInfo(parent_task, request=True)["request"][0]
         if task_request_url.startswith("git+https"):
             return task_request_url
-        raise HTTPException(
-            detail=(
-                f"Task {self.task_id}, parent task {parent_task} do not have a link to sources. "
-                "We can't locate the specfile."
-            ),
-            status_code=HTTPStatus.BAD_REQUEST,
-        )
+        return None
 
     def _fetch_task_logs_from_task_id(self) -> list[dict[str, str]]:
-        if self.task_info["arch"] != self.arch:
-            raise HTTPException(
-                detail=(
-                    f"Bad arch of task {self.task_id}: "
-                    f'expected: {self.arch} actual: {self.task_info["arch"]}'
-                ),
-                status_code=HTTPStatus.BAD_REQUEST,
-            )
+        # since we require arch in the input, we can check if the task matches it
+        # but I think it's not a good UX, if the user gives us task ID, let's just use it
+        # if someone complains about, just reintroduce the if below
+        # if self.task_info["arch"] != self.arch:
 
         if self.task_info["method"] not in ("buildArch", "buildSRPMFromSCM"):
+            # we could navigate to the right task, but let's be explicit in the meantime
+            # let user input the proper task instead of us guessing
             raise HTTPException(
                 detail=(
                     f"Task {self.task_id} method is "
@@ -287,24 +284,71 @@ def fetch_logs(self) -> list[dict[str, str]]:
 
         return logs
 
+    def _get_srpm_url_from_task(self) -> Optional[str]:
+        # example: 'cli-build/1705395313.3717997.mjCDejui/sqlite-3.45.0-1.fc40.src.rpm'
+        request_endpoint = self.task_request[0]
+        if not request_endpoint.endswith(".src.rpm"):
+            logging.error(f"Cannot find SRPM for task {self.task_id}.")
+            return None
+        return f"{self.koji_pkgs_url}/{request_endpoint}"
+
+    @staticmethod
+    def _get_spec_file_content_from_srpm(
+        srpm_path: Path, temp_dir: Path
+    ) -> Optional[dict[str, str]]:
+        # extract spec file from srpm
+        cmd = f"rpm2archive -n < {str(srpm_path)} | tar xf - '*.spec'"
+        subprocess.run(cmd, shell=True, check=True, cwd=temp_dir, capture_output=True)
+        fst_spec_file = next(temp_dir.glob("*.spec"), None)
+        if fst_spec_file is None:
+            return None
+
+        with open(fst_spec_file) as spec_file:
+            return {"name": fst_spec_file.name, "content": spec_file.read()}
+
+    def _fetch_spec_file_from_task_id(self) -> Optional[dict[str, str]]:
+        with get_temporary_dir() as temp_dir:
+            srpm_url = self._get_srpm_url_from_task()
+            if not srpm_url:
+                return None
+            resp = requests.get(srpm_url)
+            if not resp.ok:
+                logging.error(f"SRPM {srpm_url} for task {self.task_id} not "
+                              f"accessible: {resp.status_code} ({resp.reason})")
+                return None
+
+            destination = Path(f'{temp_dir}/{srpm_url.split("/")[-1]}')
+            with open(destination, "wb") as srpm_f:
+                srpm_f.write(resp.content)
+
+            return self._get_spec_file_content_from_srpm(destination, temp_dir)
+
     @handle_errors
-    def fetch_spec_file(self) -> dict[str, str]:
+    def fetch_spec_file(self) -> Optional[dict[str, str]]:
+        """
+        Fetch spec file from dist-git if possible.
+
+        Otherwise, download the SRPM and extract spec out of it.
+        """
+        request_url = self.get_task_request_url()
+        # request_url is not a link but rather a relative path to the SRPM
+        if request_url is None:
+            return self._fetch_spec_file_from_task_id()
+        package_name = re.findall(r"/rpms/(.+)\.git", request_url)[0]
+        commit_hash = re.findall(r"\.git#(.+)$", request_url)[0]
+        spec_url = "https://src.fedoraproject.org/rpms/" \
+                   f"{package_name}/raw/{commit_hash}/f/{package_name}.spec"
+        response = requests.get(spec_url)
         try:
-            request_url = self.get_task_request_url()
-            package_name = re.findall(r"/rpms/(.+)\.git", request_url)[0]
-            commit_hash = re.findall(r"\.git#(.+)$", request_url)[0]
-            spec_url = "https://src.fedoraproject.org/rpms/" \
-                f"{package_name}/raw/{commit_hash}/f/{package_name}.spec"
-            response = requests.get(spec_url)
             response.raise_for_status()
-            spec_dict = {"name": f"{package_name}.spec", "content": response.text}
         except HTTPError as exc:
-            raise FetchError(
+            logging.error(
                 "No spec file found in koji for task "
                 f"#{self.task_id} and arch {self.arch}."
                 f"Reason: {exc}"
-            ) from exc
-        return spec_dict
+            )
+            return None
+        return {"name": f"{package_name}.spec", "content": response.text}
 
 
 class PackitProvider(RPMProvider):