Move uploads directory configuration to storage API
Make use of the MEDIA_ROOT setting to store logfiles for tests in
dedicated test directories. Further adapt the logfile path migration to
the use of the storage API.
Issue: HIC-240
Change-Id: I6dd3048f9987f1185fe124f673a0b0ac7ac101e1
diff --git a/crashreport_stats/tests/test_rest_endpoints.py b/crashreport_stats/tests/test_rest_endpoints.py
index 7cef128..844e131 100644
--- a/crashreport_stats/tests/test_rest_endpoints.py
+++ b/crashreport_stats/tests/test_rest_endpoints.py
@@ -1,10 +1,10 @@
"""Tests for the rest_endpoints module."""
-
from datetime import datetime, timedelta
import operator
import unittest
import pytz
+from django.test import override_settings
from django.urls import reverse
from django.utils.http import urlencode
@@ -15,6 +15,7 @@
from crashreport_stats.tests.utils import Dummy, HiccupStatsAPITestCase
from crashreports.models import Crashreport, HeartBeat, LogFile
+from crashreports.tests.utils import DEFAULT_DUMMY_LOG_FILE_DIRECTORY
# pylint: disable=too-many-public-methods
@@ -355,6 +356,7 @@
return Dummy.create_dummy_daily_radio_version(version, **kwargs)
+@override_settings(MEDIA_ROOT=DEFAULT_DUMMY_LOG_FILE_DIRECTORY)
class DeviceStatsTestCase(HiccupStatsAPITestCase):
"""Test the single device stats REST endpoints."""
diff --git a/crashreport_stats/tests/utils.py b/crashreport_stats/tests/utils.py
index d8f0d9d..9157a63 100644
--- a/crashreport_stats/tests/utils.py
+++ b/crashreport_stats/tests/utils.py
@@ -1,7 +1,6 @@
"""Utility functions shared by all crashreport stats tests."""
from datetime import datetime, date
-import os
import zipfile
import pytz
@@ -106,7 +105,7 @@
DEFAULT_DUMMY_LOG_FILE_VALUES = {
"logfile_type": "last_kmsg",
- "logfile": os.path.join("resources", "test", "test_logfile.zip"),
+ "logfile": "test_logfile.zip",
}
DEFAULT_DUMMY_LOG_FILE_NAME = "dmesg.log"
diff --git a/crashreports/migrations/0004_update_logfile_paths.py b/crashreports/migrations/0004_update_logfile_paths.py
index fe6afac..bebdfe2 100644
--- a/crashreports/migrations/0004_update_logfile_paths.py
+++ b/crashreports/migrations/0004_update_logfile_paths.py
@@ -9,6 +9,7 @@
from django.db import migrations
from django.conf import settings
+from django.core.files.storage import default_storage
from crashreports.models import LogFile, crashreport_file_name
@@ -26,16 +27,24 @@
logger = get_django_logger()
crashreport_uploads_dir = "crashreport_uploads"
- crashreport_uploads_legacy_dir = "crashreport_uploads_legacy"
- if not os.path.isdir(crashreport_uploads_dir):
+
+ if not LogFile.objects.filter(
+ logfile__startswith=crashreport_uploads_dir
+ ).exists():
logger.info(
- "%s directory not found. Assuming this is a new installation and "
- "the migration does not need to be applied.",
- os.path.join(settings.BASE_DIR, crashreport_uploads_dir),
+ "No old logfile path found. Assuming this is a new installation "
+ "and the migration does not need to be applied."
)
return
- shutil.move(crashreport_uploads_dir, crashreport_uploads_legacy_dir)
+ crashreport_uploads_legacy_dir = crashreport_uploads_dir + "_legacy"
+ assert not os.path.isdir(crashreport_uploads_legacy_dir), (
+ "Existing crashreport_uploads_legacy directory found. Remove this"
+ "directory in order to run this migration."
+ )
+
+ if os.path.isdir(crashreport_uploads_dir):
+ shutil.move(crashreport_uploads_dir, crashreport_uploads_legacy_dir)
for logfile in LogFile.objects.all():
migrate_logfile_instance(
@@ -49,30 +58,33 @@
"""Migrate a single logfile instance."""
logger = get_django_logger()
- original_path = logfile.logfile.name
- old_logfile_path = original_path.replace(
+ old_logfile_relative_path = logfile.logfile.name.replace(
crashreport_uploads_dir, crashreport_uploads_legacy_dir, 1
)
- new_logfile_path = crashreport_file_name(
- logfile, os.path.basename(original_path)
+ old_logfile_absolute_path = os.path.join(
+ settings.BASE_DIR, old_logfile_relative_path
)
- logger.info("Migrating %s", old_logfile_path)
- if os.path.isfile(old_logfile_path):
+ new_logfile_path = crashreport_file_name(
+ logfile, os.path.basename(old_logfile_relative_path)
+ )
+ logger.info("Migrating %s", old_logfile_absolute_path)
+ if os.path.isfile(old_logfile_absolute_path):
update_logfile_path(logfile, new_logfile_path)
- move_logfile_file(old_logfile_path, new_logfile_path)
+ move_logfile_file(old_logfile_absolute_path, new_logfile_path)
else:
- logger.warning("Logfile does not exist: %s", old_logfile_path)
+ logger.warning("Logfile does not exist: %s", old_logfile_absolute_path)
def move_logfile_file(old_logfile_path, new_logfile_path):
"""Move a logfile to a new path and delete empty directories."""
logger = get_django_logger()
+ new_logfile_absolute_path = default_storage.path(new_logfile_path)
- logger.debug("Creating directories for %s", new_logfile_path)
- os.makedirs(os.path.dirname(new_logfile_path), exist_ok=True)
+ logger.debug("Creating directories for %s", new_logfile_absolute_path)
+ os.makedirs(os.path.dirname(new_logfile_absolute_path), exist_ok=True)
- logger.debug("Moving %s to %s", old_logfile_path, new_logfile_path)
- shutil.move(old_logfile_path, new_logfile_path)
+ logger.debug("Moving %s to %s", old_logfile_path, new_logfile_absolute_path)
+ shutil.move(old_logfile_path, new_logfile_absolute_path)
logger.debug("Deleting empty directories from %s", old_logfile_path)
os.removedirs(os.path.dirname(old_logfile_path))
diff --git a/crashreports/models.py b/crashreports/models.py
index 318ddac..c54f7f1 100644
--- a/crashreports/models.py
+++ b/crashreports/models.py
@@ -71,7 +71,6 @@
"""
return os.path.join(
- "crashreport_uploads",
str(instance.crashreport.device.uuid)[0:2],
str(instance.crashreport.device.uuid)[2:4],
str(instance.crashreport.device.uuid)[4:],
diff --git a/crashreports/tests/test_rest_api_logfiles.py b/crashreports/tests/test_rest_api_logfiles.py
index 09eb7af..56c6f1c 100644
--- a/crashreports/tests/test_rest_api_logfiles.py
+++ b/crashreports/tests/test_rest_api_logfiles.py
@@ -1,8 +1,13 @@
"""Tests for the logfiles REST API."""
import os
+import shutil
+import tempfile
import zipfile
+from django.conf import settings
+from django.core.files.storage import default_storage
+from django.test import override_settings
from django.urls import reverse
from rest_framework import status
@@ -11,6 +16,7 @@
from crashreports.tests.utils import HiccupCrashreportsAPITestCase, Dummy
+@override_settings(MEDIA_ROOT=tempfile.mkdtemp(".hiccup-tests"))
class LogfileUploadTest(HiccupCrashreportsAPITestCase):
"""Test cases for upload of log files."""
@@ -80,11 +86,12 @@
logfile_instance, logfile_name
)
- self.assertTrue(os.path.isfile(uploaded_logfile_path))
+ self.assertTrue(default_storage.exists(uploaded_logfile_path))
# The files are not 100% equal, because the server adds some extra
# bytes. However, we mainly care that the contents are equal:
self._assert_zip_file_contents_equal(
- uploaded_logfile_path, Dummy.DEFAULT_DUMMY_LOG_FILE_PATH
+ default_storage.path(uploaded_logfile_path),
+ Dummy.DEFAULT_DUMMY_LOG_FILE_PATH,
)
def test_logfile_upload_as_user(self):
@@ -97,15 +104,4 @@
def tearDown(self):
"""Remove the file and directories that were created for the test."""
- device = Device.objects.get(uuid=self.device_uuid)
- for crashreport_instance in device.crashreports.all():
- for logfile_instance in crashreport_instance.logfiles.all():
- uploaded_logfile_path = crashreport_file_name(
- logfile_instance,
- os.path.basename(Dummy.DEFAULT_DUMMY_LOG_FILE_PATH),
- )
- try:
- os.remove(uploaded_logfile_path)
- os.removedirs(os.path.dirname(uploaded_logfile_path))
- except (FileNotFoundError, OSError):
- pass
+ shutil.rmtree(settings.MEDIA_ROOT)
diff --git a/crashreports/tests/utils.py b/crashreports/tests/utils.py
index ac59231..15ace6b 100644
--- a/crashreports/tests/utils.py
+++ b/crashreports/tests/utils.py
@@ -10,6 +10,8 @@
from crashreports.models import Crashreport
+DEFAULT_DUMMY_LOG_FILE_DIRECTORY = os.path.join("resources", "test")
+
class InvalidCrashTypeError(BaseException):
"""Invalid crash type encountered.
@@ -74,7 +76,7 @@
}
DEFAULT_DUMMY_LOG_FILE_PATH = os.path.join(
- "resources", "test", "test_logfile.zip"
+ DEFAULT_DUMMY_LOG_FILE_DIRECTORY, "test_logfile.zip"
)
@staticmethod
diff --git a/hiccup/settings.py b/hiccup/settings.py
index fc3e2d1..0b09890 100644
--- a/hiccup/settings.py
+++ b/hiccup/settings.py
@@ -185,6 +185,8 @@
STATIC_ROOT = os.path.join(BASE_DIR, "static")
+MEDIA_ROOT = os.path.join(BASE_DIR, "crashreport_uploads")
+
# Logging
# https://docs.djangoproject.com/en/2.0/topics/logging/#examples