backend: move backend code to src folder to make space for tests

and also for other files

.github/workflows/backend-tests.yml

@@ -0,0 +1,23 @@
+name: Test backend
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  test-backend:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install dependencies
+        run: |
+          sudo apt-get install make
+      - name: Test backend inside container
+        run: |
+          make test-backend-in-container
+        shell: bash

Makefile

@@ -1,4 +1,8 @@
 COMPOSE ?= $(shell command -v podman-compose 2> /dev/null || echo docker-compose)
+CONTAINER_ENGINE ?= $(shell command -v podman 2> /dev/null || echo docker)
+TEST_IMAGE_NAME=log-detective-website_test-backend
+FEEDBACK_DIR=/persistent/results
+PYTHONPATH=/opt/log-detective-website/backend
 
 
 build-prod:
@@ -10,3 +14,21 @@ up-prod:
 # don't forget to log in to quay.io
 push-prod:
 	$(COMPOSE) -f docker-compose.prod.yaml push
+
+
+# take care of the pre-requisites by yourself
+test-backend-local:
+	PYTHONPATH=./backend pytest -vvv backend/tests/
+
+
+test-backend-in-container:
+	$(CONTAINER_ENGINE) build --rm --tag $(TEST_IMAGE_NAME) \
+		-f docker/backend/Dockerfile \
+		-f docker/backend/Dockerfile.tests
+	$(CONTAINER_ENGINE) run -t -i \
+		-e PYTHONPATH="$(PYTHONPATH)" \
+		-e FEEDBACK_DIR="$(FEEDBACK_DIR)" \
+		-e ENV="devel" \
+		-v .:/opt/log-detective-website:z \
+		$(TEST_IMAGE_NAME) \
+		bash -c "pytest -vvv /opt/log-detective-website/backend/tests/" \

backend/.gitignore

@@ -9,6 +9,8 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
+# except test data
+!tests/unit/test_data/build/
 develop-eggs/
 dist/
 downloads/
@@ -31,6 +33,8 @@ MANIFEST
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
+# except test fake spec
+!tests/unit/test_data/fake.spec
 
 # Installer logs
 pip-log.txt

backend/README.md

@@ -17,4 +17,25 @@ Run the development ASGI server with
 PYTHONPATH=/opt/log-detective-website uvicorn api:app --host 0.0.0.0 --port 5020 --reload
 ```
 
+or use compose:
+
+```bash
+docker-compose up -d
+```
+
 Open http://127.0.0.1:5020 in your web browser.
+
+## Testing
+
+Run the tests in container with
+
+```bash
+make test-backend-in-container
+```
+
+or locally on your machine. You need to have installed all the dependencies inside all dockerfiles
+located in the docker/backend directory.
+
+```bash
+make test-backend-local
+```

backend/api.py → backend/src/api.py

@@ -15,36 +15,37 @@
 from pydantic import BaseModel
 from starlette.exceptions import HTTPException
 
-from backend.constants import (
+from src.constants import (
     COPR_BUILD_URL,
     KOJI_BUILD_URL,
     PACKIT_BUILD_URL,
     FEEDBACK_DIR,
     BuildIdTitleEnum,
     ProvidersEnum,
 )
-from backend.fetcher import (
+from src.fetcher import (
     ContainerProvider,
     CoprProvider,
     KojiProvider,
     PackitProvider,
     URLProvider,
     fetch_debug_logs,
 )
-from backend.schema import (
+from src.schema import (
     ContributeResponseSchema,
     FeedbackInputSchema,
     FeedbackSchema,
     schema_inp_to_out,
 )
-from backend.spells import make_tar, get_temporary_dir
-from backend.store import Storator3000
+from src.spells import make_tar, get_temporary_dir
+from src.store import Storator3000
 
 logger = logging.getLogger(__name__)
 
 app = FastAPI()
 
-template_dir = "../frontend/public"
+# TODO: use absolute path perhaps?
+template_dir = "../../frontend/public"
 app.mount("/static", StaticFiles(directory=template_dir), name="static")
 # blame scarlette for not being able to mount directories recursively
 for root, directories, _ in os.walk(template_dir):

backend/fetcher.py → backend/src/fetcher.py

@@ -15,10 +15,10 @@
 import requests
 from fastapi import HTTPException
 
-from backend.constants import COPR_RESULT_TEMPLATE
-from backend.data import LOG_OUTPUT
-from backend.exceptions import FetchError
-from backend.spells import get_temporary_dir
+from src.constants import COPR_RESULT_TEMPLATE
+from src.data import LOG_OUTPUT
+from src.exceptions import FetchError
+from src.spells import get_temporary_dir
 
 
 def handle_errors(func):

backend/schema.py → backend/src/schema.py

@@ -2,7 +2,7 @@
 
 from pydantic import AnyUrl, BaseModel, root_validator
 
-from backend.constants import BuildIdTitleEnum
+from src.constants import BuildIdTitleEnum
 
 
 def _check_spec_container_are_exclusively_mutual(_, values):

backend/store.py → backend/src/store.py

@@ -5,9 +5,9 @@
 from pathlib import Path
 from itertools import chain
 
-from backend.constants import FEEDBACK_DIR, ProvidersEnum
-from backend.exceptions import NoDataFound
-from backend.schema import FeedbackSchema
+from src.constants import FEEDBACK_DIR, ProvidersEnum
+from src.exceptions import NoDataFound
+from src.schema import FeedbackSchema
 
 
 class Storator3000:

backend/tests/conftest.py

@@ -0,0 +1,181 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from src.constants import ProvidersEnum
+from src.schema import FeedbackInputSchema, FeedbackSchema
+from src.store import Storator3000
+
+
+FAKE_BUILD_LOG = """
+Mock Version: 5.5
+...
+"""
+
+FAKE_BACKEND_LOG = """
+[2024-02-22 13:52:52,272][  INFO][PID:1055823] Marking build as starting
+[2024-02-22 13:52:52,329][  INFO][PID:1055823] Trying to allocate VM
+[2024-02-22 13:52:55,354][  INFO][PID:1055823] Allocated host ResallocHost, ticket_id=424242
+[2024-02-22 13:52:55,354][  INFO][PID:1055823] Allocating ssh connection to builder
+...
+"""
+
+FAKE_BUILDER_LIVE_LOG = """
+Warning: Permanently added '2620:52:3:1:dead:beef:cafe:c196' (ED25519) to the list of known hosts.
+
+You can reproduce this build on your computer by running:
+
+  sudo dnf install copr-rpmbuild
+...
+"""
+
+PARENT_DIR_PATH = Path(__file__).parent
+
+
+# task_id: 114607543
+@pytest.fixture
+def srpm_task_dict():
+    with open(PARENT_DIR_PATH / "unit/test_data/task/srpm_task_dict.json") as f:
+        yield json.load(f)
+
+
+# task_id: 114656851
+@pytest.fixture
+def rpm_build_noarch_task_dict():
+    with open(PARENT_DIR_PATH / "unit/test_data/task/rpm_build_noarch_task_dict.json") as f:
+        yield json.load(f)
+
+
+# task_id: 114657791
+@pytest.fixture
+def rpm_build_arch_task_dict():
+    with open(PARENT_DIR_PATH / "unit/test_data/task/rpm_build_arch_task_dict.json") as f:
+        yield json.load(f)
+
+
+@pytest.fixture
+def copr_build_dict():
+    with open(PARENT_DIR_PATH / "unit/test_data/build/copr_build.json") as f:
+        yield json.load(f)
+
+
+@pytest.fixture
+def copr_task_descendants():
+    with open(PARENT_DIR_PATH / "unit/test_data/build/task_descendants.json") as f:
+        yield json.load(f)
+
+
+@pytest.fixture()
+def fake_spec_file():
+    with open(PARENT_DIR_PATH / "unit/test_data/fake.spec") as f:
+        yield f.read()
+
+
+@pytest.fixture()
+def copr_chroot_logs():
+    yield {
+        "build.log.gz": FAKE_BUILD_LOG,
+        "backend.log.gz": FAKE_BACKEND_LOG,
+        "builder-live.log.gz": FAKE_BUILDER_LIVE_LOG,
+    }
+
+
+@pytest.fixture()
+def copr_srpm_logs():
+    yield {
+        "backend.log.gz": FAKE_BACKEND_LOG,
+        "builder-live.log.gz": FAKE_BUILDER_LIVE_LOG,
+    }
+
+
+@pytest.fixture()
+def koji_chroot_logs_x86_64():
+    prefix = "./backend/tests/files/koji"
+    with (
+        open(f"{prefix}/build_x86_64.log") as f_build_log,
+        open(f"{prefix}/mock_output_x86_64.log") as f_mock_output_log,
+        open(f"{prefix}/root_x86_64.log") as f_root_log,
+    ):
+        yield {
+            "build.log": f_build_log.read(),
+            "mock_output.log": f_mock_output_log.read(),
+            "root.log": f_root_log.read(),
+        }
+
+
+@pytest.fixture
+def storator():
+    return Storator3000(ProvidersEnum.copr, 123)
+
+
+@pytest.fixture
+def spec_feedback_input_output_schema_tuple():
+    yield FeedbackInputSchema(
+        username="john_doe",
+        logs=[
+            {
+                "name": "log1",
+                "content": "log content 1",
+                "snippets": [
+                    {"start_index": 1, "end_index": 2, "user_comment": "comment1"}
+                ],
+            },
+        ],
+        fail_reason="Some reason",
+        how_to_fix="Some instructions",
+        spec_file={"name": "spec", "content": "spec content"},
+    ), FeedbackSchema(
+        username="john_doe",
+        logs={
+            "log1": {
+                "name": "log1",
+                "content": "log content 1",
+                "snippets": [
+                    {"start_index": 1, "end_index": 2, "user_comment": "comment1"}
+                ],
+            }
+        },
+        fail_reason="Some reason",
+        how_to_fix="Some instructions",
+        spec_file={"name": "spec", "content": "spec content"},
+    )
+
+
+@pytest.fixture
+def container_feedback_input_output_schema_tuple():
+    yield FeedbackInputSchema(
+        username="john_doe",
+        logs=[
+            {
+                "name": "log1",
+                "content": "log content 1",
+                "snippets": [
+                    {"start_index": 1, "end_index": 2, "user_comment": "comment1"}
+                ],
+            },
+        ],
+        fail_reason="Some reason",
+        how_to_fix="Some instructions",
+        container_file={
+            "name": "container_file",
+            "content": "container_file content",
+        },
+    ), FeedbackSchema(
+        username="john_doe",
+        logs={
+            "log1": {
+                "name": "log1",
+                "content": "log content 1",
+                "snippets": [
+                    {"start_index": 1, "end_index": 2, "user_comment": "comment1"}
+                ],
+            }
+        },
+        fail_reason="Some reason",
+        how_to_fix="Some instructions",
+        container_file={
+            "name": "container_file",
+            "content": "container_file content",
+        },
+    )

backend/tests/spells.py

@@ -0,0 +1,12 @@
+"""Lyney would approve this :magic:"""
+
+import responses
+
+
+def mock_multiple_responses(url, logs):
+    for name, content in logs.items():
+        responses.add(responses.GET, f"{url}/{name}", body=content, status=200)
+
+
+def sort_by_name(item):
+    return item["name"]

backend/tests/unit/test_data/build/copr_build.json

@@ -0,0 +1 @@
+{"build_id": 2414873, "cg_id": null, "completion_time": "2024-03-04 11:50:16.549582+00:00", "completion_ts": 1709553016.549582, "creation_event_id": 127221990, "creation_time": "2024-03-04 11:42:14.519109+00:00", "creation_ts": 1709552534.519109, "draft": false, "epoch": null, "extra": {"source": {"original_url": "git+https://src.fedoraproject.org/rpms/copr-frontend.git#dbcd207295f92021a8528a3b4bbdff697e317e12"}}, "id": 2414873, "name": "copr-frontend", "nvr": "copr-frontend-1.204-1.fc40", "owner_id": 1823, "owner_name": "praiskup", "package_id": 18554, "package_name": "copr-frontend", "promoter_id": null, "promoter_name": null, "promotion_time": null, "promotion_ts": null, "release": "1.fc40", "source": "git+https://src.fedoraproject.org/rpms/copr-frontend.git#dbcd207295f92021a8528a3b4bbdff697e317e12", "start_time": "2024-03-04 11:42:14.514327+00:00", "start_ts": 1709552534.514327, "state": 3, "task_id": 114443211, "version": "1.204", "volume_id": 0, "volume_name": "DEFAULT", "cg_name": null}

backend/tests/unit/test_data/build/task_descendants.json

@@ -0,0 +1 @@
+[{"arch": "noarch", "awaited": false, "channel_id": 2, "completion_time": "2024-03-04 11:41:59.299605+00:00", "completion_ts": 1709552519.299605, "create_time": "2024-03-04 11:39:34.409462+00:00", "create_ts": 1709552374.409462, "host_id": 381, "id": 114443241, "label": "srpm", "method": "buildSRPMFromSCM", "owner": 1823, "parent": 114443211, "priority": 19, "start_time": "2024-03-04 11:40:34.981490+00:00", "start_ts": 1709552434.98149, "state": 2, "waiting": null, "weight": 1.0}, {"arch": "noarch", "awaited": false, "channel_id": 1, "completion_time": "2024-03-04 11:50:06.780025+00:00", "completion_ts": 1709553006.780025, "create_time": "2024-03-04 11:42:14.782312+00:00", "create_ts": 1709552534.782312, "host_id": 523, "id": 114443338, "label": "noarch", "method": "buildArch", "owner": 1823, "parent": 114443211, "priority": 19, "start_time": "2024-03-04 11:42:40.316773+00:00", "start_ts": 1709552560.316773, "state": 5, "waiting": null, "weight": 1.542257300625}]

backend/tests/unit/test_data/fake.spec

@@ -0,0 +1,17 @@
+Name:           test-package
+Version:        0.1.0
+Release:        1%{?dist}
+Summary:        Fake test package
+
+License:        GPLv3
+URL:            https://example.com/%{name}
+Source0:        %{url}/%{version}.tar.gz
+
+
+%description
+%{summary}
+
+
+%changelog
+* Fri May 12 2023 [email protected]
+- initial release