Addon startup waits for healthy (#4376)

* Addon startup waits for healthy

* fix import for pylint

* wait_for to 5 in tests

* Adjust tests to simplify async tasks

* Remove wait_boot time from addons.boot tests

* Eliminate async task race conditions in tests
This commit is contained in:
Mike Degatano 2023-06-20 10:13:15 -04:00 committed by GitHub
parent e4ee3e4226
commit 254ec2d1af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 846 additions and 65 deletions

View File

@ -1,5 +1,6 @@
"""Init file for Supervisor add-ons."""
import asyncio
from collections.abc import Awaitable
from contextlib import suppress
import logging
import tarfile
@ -104,9 +105,13 @@ class AddonManager(CoreSysAttributes):
# Start Add-ons sequential
# avoid issue on slow IO
# Config.wait_boot is deprecated. Until addons update with healthchecks,
# add a sleep task for it to keep the same minimum amount of wait time
wait_boot: list[Awaitable[None]] = [asyncio.sleep(self.sys_config.wait_boot)]
for addon in tasks:
try:
await addon.start()
if start_task := await addon.start():
wait_boot.append(start_task)
except AddonsError as err:
# Check if there is an system/user issue
if check_exception_chain(
@ -121,7 +126,8 @@ class AddonManager(CoreSysAttributes):
_LOGGER.warning("Can't start Add-on %s", addon.slug)
await asyncio.sleep(self.sys_config.wait_boot)
# Ignore exceptions from waiting for addon startup, addon errors handled elsewhere
await asyncio.gather(*wait_boot, return_exceptions=True)
async def shutdown(self, stage: AddonStartup) -> None:
"""Shutdown addons."""
@ -244,8 +250,14 @@ class AddonManager(CoreSysAttributes):
conditions=ADDON_UPDATE_CONDITIONS,
on_condition=AddonsJobError,
)
async def update(self, slug: str, backup: bool | None = False) -> None:
"""Update add-on."""
async def update(
self, slug: str, backup: bool | None = False
) -> Awaitable[None] | None:
"""Update add-on.
Returns a coroutine that completes when addon has state 'started' (see addon.start)
if addon is started after update. Else nothing is returned.
"""
if slug not in self.local:
raise AddonsError(f"Add-on {slug} is not installed", _LOGGER.error)
addon = self.local[slug]
@ -288,8 +300,11 @@ class AddonManager(CoreSysAttributes):
await addon.install_apparmor()
# restore state
if last_state == AddonState.STARTED:
return (
await addon.start()
if last_state in [AddonState.STARTED, AddonState.STARTUP]
else None
)
@Job(
conditions=[
@ -299,8 +314,12 @@ class AddonManager(CoreSysAttributes):
],
on_condition=AddonsJobError,
)
async def rebuild(self, slug: str) -> None:
"""Perform a rebuild of local build add-on."""
async def rebuild(self, slug: str) -> Awaitable[None] | None:
"""Perform a rebuild of local build add-on.
Returns a coroutine that completes when addon has state 'started' (see addon.start)
if addon is started after rebuild. Else nothing is returned.
"""
if slug not in self.local:
raise AddonsError(f"Add-on {slug} is not installed", _LOGGER.error)
addon = self.local[slug]
@ -333,8 +352,11 @@ class AddonManager(CoreSysAttributes):
_LOGGER.info("Add-on '%s' successfully rebuilt", slug)
# restore state
if last_state == AddonState.STARTED:
return (
await addon.start()
if last_state in [AddonState.STARTED, AddonState.STARTUP]
else None
)
@Job(
conditions=[
@ -344,8 +366,14 @@ class AddonManager(CoreSysAttributes):
],
on_condition=AddonsJobError,
)
async def restore(self, slug: str, tar_file: tarfile.TarFile) -> None:
"""Restore state of an add-on."""
async def restore(
self, slug: str, tar_file: tarfile.TarFile
) -> Awaitable[None] | None:
"""Restore state of an add-on.
Returns a coroutine that completes when addon has state 'started' (see addon.start)
if addon is started after restore. Else nothing is returned.
"""
if slug not in self.local:
_LOGGER.debug("Add-on %s is not local available for restore", slug)
addon = Addon(self.coresys, slug)
@ -353,7 +381,7 @@ class AddonManager(CoreSysAttributes):
_LOGGER.debug("Add-on %s is local available for restore", slug)
addon = self.local[slug]
await addon.restore(tar_file)
wait_for_start = await addon.restore(tar_file)
# Check if new
if slug not in self.local:
@ -366,6 +394,8 @@ class AddonManager(CoreSysAttributes):
with suppress(HomeAssistantAPIError):
await self.sys_ingress.update_hass_panel(addon)
return wait_for_start
@Job(conditions=[JobCondition.FREE_SPACE, JobCondition.INTERNET_HOST])
async def repair(self) -> None:
"""Repair local add-ons."""

View File

@ -99,6 +99,7 @@ RE_WATCHDOG = re.compile(
)
WATCHDOG_TIMEOUT = aiohttp.ClientTimeout(total=10)
STARTUP_TIMEOUT = 120
_OPTIONS_MERGER: Final = Merger(
type_strategies=[(dict, ["merge"])],
@ -106,6 +107,14 @@ _OPTIONS_MERGER: Final = Merger(
type_conflict_strategies=["override"],
)
# Backups just need to know if an addon was running or not
# Map other addon states to those two
_MAP_ADDON_STATE = {
AddonState.STARTUP: AddonState.STARTED,
AddonState.ERROR: AddonState.STOPPED,
AddonState.UNKNOWN: AddonState.STOPPED,
}
class Addon(AddonModel):
"""Hold data for add-on inside Supervisor."""
@ -119,6 +128,7 @@ class Addon(AddonModel):
self.sys_hardware.helper.last_boot != self.sys_config.last_boot
)
self._listeners: list[EventListener] = []
self._startup_event = asyncio.Event()
@Job(
name=f"addon_{slug}_restart_after_problem",
@ -144,9 +154,9 @@ class Addon(AddonModel):
with suppress(DockerError):
await addon.instance.stop(remove_container=True)
await addon.start()
await (await addon.start())
else:
await addon.restart()
await (await addon.restart())
except AddonsError as err:
attempts = attempts + 1
_LOGGER.error(
@ -182,7 +192,13 @@ class Addon(AddonModel):
"""Set the add-on into new state."""
if self._state == new_state:
return
old_state = self._state
self._state = new_state
# Signal listeners about addon state change
if new_state == AddonState.STARTED or old_state == AddonState.STARTUP:
self._startup_event.set()
self.sys_homeassistant.websocket.send_message(
{
ATTR_TYPE: WSType.SUPERVISOR_EVENT,
@ -680,8 +696,24 @@ class Addon(AddonModel):
return False
return True
async def start(self) -> None:
"""Set options and start add-on."""
async def _wait_for_startup(self) -> None:
"""Wait for startup event to be set with timeout."""
try:
await asyncio.wait_for(self._startup_event.wait(), STARTUP_TIMEOUT)
except asyncio.TimeoutError:
_LOGGER.warning(
"Timeout while waiting for addon %s to start, took more then %s seconds",
self.name,
STARTUP_TIMEOUT,
)
async def start(self) -> Awaitable[None]:
"""Set options and start add-on.
Returns a coroutine that completes when addon has state 'started'.
For addons with a healthcheck, that is when they become healthy or unhealthy.
Addons without a healthcheck have state 'started' immediately.
"""
if await self.instance.is_running():
_LOGGER.warning("%s is already running!", self.slug)
return
@ -698,12 +730,15 @@ class Addon(AddonModel):
self.write_pulse()
# Start Add-on
self._startup_event.clear()
try:
await self.instance.run()
except DockerError as err:
self.state = AddonState.ERROR
raise AddonsError() from err
return self._wait_for_startup()
async def stop(self) -> None:
"""Stop add-on."""
self._manual_stop = True
@ -713,11 +748,14 @@ class Addon(AddonModel):
self.state = AddonState.ERROR
raise AddonsError() from err
async def restart(self) -> None:
"""Restart add-on."""
async def restart(self) -> Awaitable[None]:
"""Restart add-on.
Returns a coroutine that completes when addon has state 'started' (see start).
"""
with suppress(AddonsError):
await self.stop()
await self.start()
return await self.start()
def logs(self) -> Awaitable[bytes]:
"""Return add-ons log output.
@ -772,8 +810,13 @@ class Addon(AddonModel):
_LOGGER.error,
) from err
async def backup(self, tar_file: tarfile.TarFile) -> None:
"""Backup state of an add-on."""
async def backup(self, tar_file: tarfile.TarFile) -> Awaitable[None] | None:
"""Backup state of an add-on.
Returns a coroutine that completes when addon has state 'started' (see start)
for cold backup. Else nothing is returned.
"""
wait_for_start: Awaitable[None] | None = None
is_running = await self.is_running()
with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
@ -790,7 +833,7 @@ class Addon(AddonModel):
ATTR_USER: self.persist,
ATTR_SYSTEM: self.data,
ATTR_VERSION: self.version,
ATTR_STATE: self.state,
ATTR_STATE: _MAP_ADDON_STATE.get(self.state, self.state),
}
# Store local configs/state
@ -852,12 +895,18 @@ class Addon(AddonModel):
await self._backup_command(self.backup_post)
elif is_running and self.backup_mode is AddonBackupMode.COLD:
_LOGGER.info("Starting add-on %s again", self.slug)
await self.start()
wait_for_start = await self.start()
_LOGGER.info("Finish backup for addon %s", self.slug)
return wait_for_start
async def restore(self, tar_file: tarfile.TarFile) -> None:
"""Restore state of an add-on."""
async def restore(self, tar_file: tarfile.TarFile) -> Awaitable[None] | None:
"""Restore state of an add-on.
Returns a coroutine that completes when addon has state 'started' (see start)
if addon is started after restore. Else nothing is returned.
"""
wait_for_start: Awaitable[None] | None = None
with TemporaryDirectory(dir=self.sys_config.path_tmp) as temp:
# extract backup
def _extract_tarfile():
@ -958,9 +1007,10 @@ class Addon(AddonModel):
# Run add-on
if data[ATTR_STATE] == AddonState.STARTED:
return await self.start()
wait_for_start = await self.start()
_LOGGER.info("Finished restore for add-on %s", self.slug)
return wait_for_start
def check_trust(self) -> Awaitable[None]:
"""Calculate Addon docker content trust.
@ -974,12 +1024,15 @@ class Addon(AddonModel):
if event.name != self.instance.name:
return
if event.state in [
ContainerState.RUNNING,
if event.state == ContainerState.RUNNING:
self._manual_stop = False
self.state = (
AddonState.STARTUP if self.instance.healthcheck else AddonState.STARTED
)
elif event.state in [
ContainerState.HEALTHY,
ContainerState.UNHEALTHY,
]:
self._manual_stop = False
self.state = AddonState.STARTED
elif event.state == ContainerState.STOPPED:
self.state = AddonState.STOPPED

View File

@ -673,10 +673,10 @@ class AddonModel(CoreSysAttributes, ABC):
"""Uninstall this add-on."""
return self.sys_addons.uninstall(self.slug)
def update(self, backup: bool | None = False) -> Awaitable[None]:
def update(self, backup: bool | None = False) -> Awaitable[Awaitable[None] | None]:
"""Update this add-on."""
return self.sys_addons.update(self.slug, backup=backup)
def rebuild(self) -> Awaitable[None]:
def rebuild(self) -> Awaitable[Awaitable[None] | None]:
"""Rebuild this add-on."""
return self.sys_addons.rebuild(self.slug)

View File

@ -391,10 +391,11 @@ class APIAddons(CoreSysAttributes):
return asyncio.shield(addon.uninstall())
@api_process
def start(self, request: web.Request) -> Awaitable[None]:
async def start(self, request: web.Request) -> None:
"""Start add-on."""
addon = self._extract_addon(request)
return asyncio.shield(addon.start())
if start_task := await asyncio.shield(addon.start()):
await start_task
@api_process
def stop(self, request: web.Request) -> Awaitable[None]:
@ -403,16 +404,18 @@ class APIAddons(CoreSysAttributes):
return asyncio.shield(addon.stop())
@api_process
def restart(self, request: web.Request) -> Awaitable[None]:
async def restart(self, request: web.Request) -> None:
"""Restart add-on."""
addon: Addon = self._extract_addon(request)
return asyncio.shield(addon.restart())
if start_task := await asyncio.shield(addon.restart()):
await start_task
@api_process
def rebuild(self, request: web.Request) -> Awaitable[None]:
async def rebuild(self, request: web.Request) -> None:
"""Rebuild local build add-on."""
addon = self._extract_addon(request)
return asyncio.shield(addon.rebuild())
if start_task := await asyncio.shield(addon.rebuild()):
await start_task
@api_process_raw(CONTENT_TYPE_BINARY)
def logs(self, request: web.Request) -> Awaitable[bytes]:

View File

@ -208,7 +208,10 @@ class APIStore(CoreSysAttributes):
body = await api_validate(SCHEMA_UPDATE, request)
return await asyncio.shield(addon.update(backup=body.get(ATTR_BACKUP)))
if start_task := await asyncio.shield(
addon.update(backup=body.get(ATTR_BACKUP))
):
await start_task
@api_process
async def addons_addon_info(self, request: web.Request) -> dict[str, Any]:

View File

@ -94,7 +94,6 @@ class APISupervisor(CoreSysAttributes):
ATTR_SUPPORTED: self.sys_core.supported,
ATTR_HEALTHY: self.sys_core.healthy,
ATTR_IP_ADDRESS: str(self.sys_supervisor.ip_address),
ATTR_WAIT_BOOT: self.sys_config.wait_boot,
ATTR_TIMEZONE: self.sys_config.timezone,
ATTR_LOGGING: self.sys_config.logging,
ATTR_DEBUG: self.sys_config.debug,
@ -102,6 +101,7 @@ class APISupervisor(CoreSysAttributes):
ATTR_DIAGNOSTICS: self.sys_config.diagnostics,
ATTR_AUTO_UPDATE: self.sys_updater.auto_update,
# Depricated
ATTR_WAIT_BOOT: self.sys_config.wait_boot,
ATTR_ADDONS: [
{
ATTR_NAME: addon.name,
@ -132,9 +132,6 @@ class APISupervisor(CoreSysAttributes):
if ATTR_TIMEZONE in body:
self.sys_config.timezone = body[ATTR_TIMEZONE]
if ATTR_WAIT_BOOT in body:
self.sys_config.wait_boot = body[ATTR_WAIT_BOOT]
if ATTR_DEBUG in body:
self.sys_config.debug = body[ATTR_DEBUG]
@ -156,6 +153,10 @@ class APISupervisor(CoreSysAttributes):
if ATTR_AUTO_UPDATE in body:
self.sys_updater.auto_update = body[ATTR_AUTO_UPDATE]
# Deprecated
if ATTR_WAIT_BOOT in body:
self.sys_config.wait_boot = body[ATTR_WAIT_BOOT]
# Save changes before processing addons in case of errors
self.sys_updater.save_data()
self.sys_config.save_data()

View File

@ -332,10 +332,14 @@ class Backup(CoreSysAttributes):
finally:
self._tmp.cleanup()
async def store_addons(self, addon_list: list[str]):
"""Add a list of add-ons into backup."""
async def store_addons(self, addon_list: list[str]) -> list[Awaitable[None]]:
"""Add a list of add-ons into backup.
async def _addon_save(addon: Addon):
For each addon that needs to be started after backup, returns a task which
completes when that addon has state 'started' (see addon.start).
"""
async def _addon_save(addon: Addon) -> Awaitable[None] | None:
"""Task to store an add-on into backup."""
tar_name = f"{addon.slug}.tar{'.gz' if self.compressed else ''}"
addon_file = SecureTarFile(
@ -348,7 +352,7 @@ class Backup(CoreSysAttributes):
# Take backup
try:
await addon.backup(addon_file)
start_task = await addon.backup(addon_file)
except AddonsError:
_LOGGER.error("Can't create backup for %s", addon.slug)
return
@ -363,18 +367,24 @@ class Backup(CoreSysAttributes):
}
)
return start_task
# Save Add-ons sequential
# avoid issue on slow IO
start_tasks: list[Awaitable[None]] = []
for addon in addon_list:
try:
await _addon_save(addon)
if start_task := await _addon_save(addon):
start_tasks.append(start_task)
except Exception as err: # pylint: disable=broad-except
_LOGGER.warning("Can't save Add-on %s: %s", addon.slug, err)
async def restore_addons(self, addon_list: list[str]):
return start_tasks
async def restore_addons(self, addon_list: list[str]) -> list[Awaitable[None]]:
"""Restore a list add-on from backup."""
async def _addon_restore(addon_slug: str):
async def _addon_restore(addon_slug: str) -> Awaitable[None] | None:
"""Task to restore an add-on into backup."""
tar_name = f"{addon_slug}.tar{'.gz' if self.compressed else ''}"
addon_file = SecureTarFile(
@ -392,18 +402,22 @@ class Backup(CoreSysAttributes):
# Perform a restore
try:
await self.sys_addons.restore(addon_slug, addon_file)
return await self.sys_addons.restore(addon_slug, addon_file)
except AddonsError:
_LOGGER.error("Can't restore backup %s", addon_slug)
# Save Add-ons sequential
# avoid issue on slow IO
start_tasks: list[Awaitable[None]] = []
for slug in addon_list:
try:
await _addon_restore(slug)
if start_task := await _addon_restore(slug):
start_tasks.append(start_task)
except Exception as err: # pylint: disable=broad-except
_LOGGER.warning("Can't restore Add-on %s: %s", slug, err)
return start_tasks
async def store_folders(self, folder_list: list[str]):
"""Backup Supervisor data into backup."""

View File

@ -2,7 +2,7 @@
from __future__ import annotations
import asyncio
from collections.abc import Iterable
from collections.abc import Awaitable, Iterable
import logging
from pathlib import Path
@ -190,6 +190,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
folder_list: list[str],
homeassistant: bool,
):
addon_start_tasks: list[Awaitable[None]] | None = None
try:
self.sys_core.state = CoreState.FREEZE
@ -197,7 +198,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
# Backup add-ons
if addon_list:
_LOGGER.info("Backing up %s store Add-ons", backup.slug)
await backup.store_addons(addon_list)
addon_start_tasks = await backup.store_addons(addon_list)
# HomeAssistant Folder is for v1
if homeassistant:
@ -214,6 +215,11 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
return None
else:
self._backups[backup.slug] = backup
if addon_start_tasks:
# Ignore exceptions from waiting for addon startup, addon errors handled elsewhere
await asyncio.gather(*addon_start_tasks, return_exceptions=True)
return backup
finally:
self.sys_core.state = CoreState.RUNNING
@ -300,6 +306,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
homeassistant: bool,
replace: bool,
):
addon_start_tasks: list[Awaitable[None]] | None = None
try:
task_hass: asyncio.Task | None = None
async with backup:
@ -336,7 +343,7 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
await backup.restore_repositories(replace)
_LOGGER.info("Restoring %s Add-ons", backup.slug)
await backup.restore_addons(addon_list)
addon_start_tasks = await backup.restore_addons(addon_list)
# Wait for Home Assistant Core update/downgrade
if task_hass:
@ -348,6 +355,10 @@ class BackupManager(FileConfiguration, CoreSysAttributes):
capture_exception(err)
return False
else:
if addon_start_tasks:
# Ignore exceptions from waiting for addon startup, addon errors handled elsewhere
await asyncio.gather(*addon_start_tasks, return_exceptions=True)
return True
finally:
# Do we need start Home Assistant Core?

View File

@ -395,6 +395,7 @@ class AddonStage(str, Enum):
class AddonState(str, Enum):
"""State of add-on."""
STARTUP = "startup"
STARTED = "started"
STOPPED = "stopped"
UNKNOWN = "unknown"

View File

@ -159,6 +159,11 @@ class DockerInterface(CoreSysAttributes):
# causes problems on some types of host systems.
return ["seccomp=unconfined"]
@property
def healthcheck(self) -> dict[str, Any] | None:
"""Healthcheck of instance if it has one."""
return self.meta_config.get("Healthcheck")
def _get_credentials(self, image: str) -> dict:
"""Return a dictionay with credentials for docker login."""
registry = None

View File

@ -185,7 +185,7 @@ class DockerAPI:
# Create container
try:
container = self.docker.containers.create(
container = self.containers.create(
f"{image}:{tag}", use_config_proxy=False, **kwargs
)
except docker_errors.NotFound as err:

View File

@ -1,4 +1,6 @@
"""A collection of tasks."""
import asyncio
from collections.abc import Awaitable
import logging
from ..addons.const import ADDON_UPDATE_CONDITIONS
@ -84,6 +86,7 @@ class Tasks(CoreSysAttributes):
@Job(conditions=ADDON_UPDATE_CONDITIONS + [JobCondition.RUNNING])
async def _update_addons(self):
"""Check if an update is available for an Add-on and update it."""
start_tasks: list[Awaitable[None]] = []
for addon in self.sys_addons.all:
if not addon.is_installed or not addon.auto_update:
continue
@ -101,10 +104,13 @@ class Tasks(CoreSysAttributes):
# avoid issue on slow IO
_LOGGER.info("Add-on auto update process %s", addon.slug)
try:
await addon.update(backup=True)
if start_task := await addon.update(backup=True):
start_tasks.append(start_task)
except AddonsError:
_LOGGER.error("Can't auto update Add-on %s", addon.slug)
await asyncio.gather(*start_tasks)
@Job(
conditions=[
JobCondition.AUTO_UPDATE,
@ -265,7 +271,7 @@ class Tasks(CoreSysAttributes):
_LOGGER.warning("Watchdog found a problem with %s application!", addon.slug)
try:
await addon.restart()
await (await addon.restart())
except AddonsError as err:
_LOGGER.error("%s watchdog reanimation failed with %s", addon.slug, err)
capture_exception(err)

View File

@ -41,7 +41,7 @@ class FixupAddonExecuteRebuild(FixupBase):
)
await addon.stop()
else:
await addon.restart()
await (await addon.restart())
@property
def suggestion(self) -> SuggestionType:

View File

@ -6,8 +6,12 @@ from unittest.mock import MagicMock, PropertyMock, patch
from docker.errors import DockerException
import pytest
from securetar import SecureTarFile
from supervisor.addons.addon import Addon
from supervisor.addons.const import AddonBackupMode
from supervisor.addons.model import AddonModel
from supervisor.arch import CpuArch
from supervisor.const import AddonState, BusEvent
from supervisor.coresys import CoreSys
from supervisor.docker.addon import DockerAddon
@ -17,7 +21,8 @@ from supervisor.exceptions import AddonsJobError, AudioUpdateError
from supervisor.store.repository import Repository
from supervisor.utils.dt import utcnow
from ..const import TEST_ADDON_SLUG
from tests.common import get_fixture_path
from tests.const import TEST_ADDON_SLUG
def _fire_test_event(coresys: CoreSys, name: str, state: ContainerState):
@ -131,6 +136,7 @@ async def test_addon_watchdog(coresys: CoreSys, install_addon_ssh: Addon) -> Non
await install_addon_ssh.load()
install_addon_ssh.watchdog = True
install_addon_ssh._manual_stop = False # pylint: disable=protected-access
with patch.object(Addon, "restart") as restart, patch.object(
Addon, "start"
@ -219,7 +225,7 @@ async def test_listener_attached_on_install(coresys: CoreSys, repository):
_fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
await asyncio.sleep(0)
assert coresys.addons.get(TEST_ADDON_SLUG).state == AddonState.STARTED
assert coresys.addons.get(TEST_ADDON_SLUG).state == AddonState.STARTUP
@pytest.mark.parametrize(
@ -304,3 +310,164 @@ async def test_listeners_removed_on_uninstall(
listener
not in coresys.bus._listeners[BusEvent.DOCKER_CONTAINER_STATE_CHANGE]
)
async def test_start(
coresys: CoreSys,
install_addon_ssh: Addon,
container,
tmp_supervisor_data,
path_extern,
) -> None:
"""Test starting an addon without healthcheck."""
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STOPPED
start_task = await install_addon_ssh.start()
assert start_task
_fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
await start_task
assert install_addon_ssh.state == AddonState.STARTED
@pytest.mark.parametrize("state", [ContainerState.HEALTHY, ContainerState.UNHEALTHY])
async def test_start_wait_healthcheck(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
state: ContainerState,
tmp_supervisor_data,
path_extern,
) -> None:
"""Test starting an addon with a healthcheck waits for health status."""
install_addon_ssh.path_data.mkdir()
container.attrs["Config"] = {"Healthcheck": "exists"}
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STOPPED
start_task = asyncio.create_task(await install_addon_ssh.start())
assert start_task
_fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
await asyncio.sleep(0.01)
assert not start_task.done()
assert install_addon_ssh.state == AddonState.STARTUP
_fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", state)
await asyncio.sleep(0.01)
assert start_task.done()
assert install_addon_ssh.state == AddonState.STARTED
async def test_start_timeout(
coresys: CoreSys,
install_addon_ssh: Addon,
caplog: pytest.LogCaptureFixture,
container,
tmp_supervisor_data,
path_extern,
) -> None:
"""Test starting an addon times out while waiting."""
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STOPPED
start_task = await install_addon_ssh.start()
assert start_task
caplog.clear()
with patch(
"supervisor.addons.addon.asyncio.wait_for", side_effect=asyncio.TimeoutError
):
await start_task
assert "Timeout while waiting for addon Terminal & SSH to start" in caplog.text
async def test_restart(
coresys: CoreSys,
install_addon_ssh: Addon,
container,
tmp_supervisor_data,
path_extern,
) -> None:
"""Test restarting an addon."""
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STOPPED
start_task = await install_addon_ssh.restart()
assert start_task
_fire_test_event(coresys, f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
await start_task
assert install_addon_ssh.state == AddonState.STARTED
@pytest.mark.parametrize("status", ["running", "stopped"])
async def test_backup(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
status: str,
tmp_supervisor_data,
path_extern,
) -> None:
"""Test backing up an addon."""
container.status = status
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
tarfile = SecureTarFile(coresys.config.path_tmp / "test.tar.gz", "w")
assert await install_addon_ssh.backup(tarfile) is None
@pytest.mark.parametrize("status", ["running", "stopped"])
async def test_backup_cold_mode(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
status: str,
tmp_supervisor_data,
path_extern,
) -> None:
"""Test backing up an addon in cold mode."""
container.status = status
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
tarfile = SecureTarFile(coresys.config.path_tmp / "test.tar.gz", "w")
with patch.object(
AddonModel, "backup_mode", new=PropertyMock(return_value=AddonBackupMode.COLD)
), patch.object(
DockerAddon, "_is_running", side_effect=[status == "running", False, False]
):
start_task = await install_addon_ssh.backup(tarfile)
assert bool(start_task) is (status == "running")
@pytest.mark.parametrize("status", ["running", "stopped"])
async def test_restore(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
status: str,
tmp_supervisor_data,
path_extern,
) -> None:
"""Test restoring an addon."""
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
tarfile = SecureTarFile(get_fixture_path(f"backup_local_ssh_{status}.tar.gz"), "r")
with patch.object(DockerAddon, "_is_running", return_value=False), patch.object(
CpuArch, "supported", new=PropertyMock(return_value=["aarch64"])
):
start_task = await coresys.addons.restore(TEST_ADDON_SLUG, tarfile)
assert bool(start_task) is (status == "running")

View File

@ -8,10 +8,12 @@ import pytest
from supervisor.addons.addon import Addon
from supervisor.arch import CpuArch
from supervisor.const import AddonBoot, AddonStartup, AddonState
from supervisor.const import AddonBoot, AddonStartup, AddonState, BusEvent
from supervisor.coresys import CoreSys
from supervisor.docker.addon import DockerAddon
from supervisor.docker.const import ContainerState
from supervisor.docker.interface import DockerInterface
from supervisor.docker.monitor import DockerContainerStateEvent
from supervisor.exceptions import (
AddonConfigurationError,
AddonsError,
@ -34,6 +36,12 @@ async def fixture_mock_arch_disk() -> None:
yield
@pytest.fixture(autouse=True)
async def fixture_remove_wait_boot(coresys: CoreSys) -> None:
"""Remove default wait boot time for tests."""
coresys.config.wait_boot = 0
async def test_image_added_removed_on_update(
coresys: CoreSys, install_addon_ssh: Addon
):
@ -182,3 +190,89 @@ async def test_load(
write_hosts.assert_called_once()
assert "Found 1 installed add-ons" in caplog.text
async def test_boot_waits_for_addons(
coresys: CoreSys,
install_addon_ssh: Addon,
container,
tmp_supervisor_data,
path_extern,
):
"""Test addon manager boot waits for addons."""
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STOPPED
addon_state: AddonState | None = None
async def fire_container_event(*args, **kwargs):
nonlocal addon_state
addon_state = install_addon_ssh.state
coresys.bus.fire_event(
BusEvent.DOCKER_CONTAINER_STATE_CHANGE,
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.RUNNING,
id="abc123",
time=1,
),
)
with patch.object(DockerAddon, "run", new=fire_container_event):
await coresys.addons.boot(AddonStartup.APPLICATION)
assert addon_state == AddonState.STOPPED
assert install_addon_ssh.state == AddonState.STARTED
@pytest.mark.parametrize("status", ["running", "stopped"])
async def test_update(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
status: str,
tmp_supervisor_data,
path_extern,
):
"""Test addon update."""
container.status = status
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
with patch(
"supervisor.store.data.read_json_or_yaml_file",
return_value=load_json_fixture("addon-config-add-image.json"),
):
await coresys.store.data.update()
assert install_addon_ssh.need_update is True
with patch.object(DockerInterface, "_install"), patch.object(
DockerAddon, "_is_running", return_value=False
):
start_task = await coresys.addons.update(TEST_ADDON_SLUG)
assert bool(start_task) is (status == "running")
@pytest.mark.parametrize("status", ["running", "stopped"])
async def test_rebuild(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
status: str,
tmp_supervisor_data,
path_extern,
):
"""Test addon rebuild."""
container.status = status
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
with patch.object(DockerAddon, "_build"), patch.object(
DockerAddon, "_is_running", return_value=False
), patch.object(Addon, "need_build", new=PropertyMock(return_value=True)):
start_task = await coresys.addons.rebuild(TEST_ADDON_SLUG)
assert bool(start_task) is (status == "running")

View File

@ -1,17 +1,33 @@
"""Test addons api."""
from unittest.mock import MagicMock
import asyncio
from unittest.mock import MagicMock, PropertyMock, patch
from aiohttp.test_utils import TestClient
from supervisor.addons.addon import Addon
from supervisor.addons.build import AddonBuild
from supervisor.arch import CpuArch
from supervisor.const import AddonState
from supervisor.coresys import CoreSys
from supervisor.docker.addon import DockerAddon
from supervisor.docker.const import ContainerState
from supervisor.docker.monitor import DockerContainerStateEvent
from supervisor.store.repository import Repository
from ..const import TEST_ADDON_SLUG
def _create_test_event(name: str, state: ContainerState) -> DockerContainerStateEvent:
"""Create a container state event."""
return DockerContainerStateEvent(
name=name,
state=state,
id="abc123",
time=1,
)
async def test_addons_info(
api_client: TestClient, coresys: CoreSys, install_addon_ssh: Addon
):
@ -62,3 +78,137 @@ async def test_api_addon_logs(
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os\x1b[0m",
"\x1b[36m22-10-11 14:04:23 DEBUG (MainThread) [supervisor.utils.dbus] D-Bus call - org.freedesktop.DBus.Properties.call_get_all on /io/hass/os/AppArmor\x1b[0m",
]
async def test_api_addon_start_healthcheck(
api_client: TestClient,
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
tmp_supervisor_data,
path_extern,
):
"""Test starting an addon waits for healthy."""
install_addon_ssh.path_data.mkdir()
container.attrs["Config"] = {"Healthcheck": "exists"}
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STOPPED
state_changes: list[AddonState] = []
async def container_events():
nonlocal state_changes
await asyncio.sleep(0.01)
await install_addon_ssh.container_state_changed(
_create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
_create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.HEALTHY)
)
async def container_events_task(*args, **kwargs):
asyncio.create_task(container_events())
with patch.object(DockerAddon, "run", new=container_events_task):
resp = await api_client.post("/addons/local_ssh/start")
assert state_changes == [AddonState.STARTUP]
assert install_addon_ssh.state == AddonState.STARTED
assert resp.status == 200
async def test_api_addon_restart_healthcheck(
api_client: TestClient,
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
tmp_supervisor_data,
path_extern,
):
"""Test restarting an addon waits for healthy."""
install_addon_ssh.path_data.mkdir()
container.attrs["Config"] = {"Healthcheck": "exists"}
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STOPPED
state_changes: list[AddonState] = []
async def container_events():
nonlocal state_changes
await asyncio.sleep(0.01)
await install_addon_ssh.container_state_changed(
_create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
_create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.HEALTHY)
)
async def container_events_task(*args, **kwargs):
asyncio.create_task(container_events())
with patch.object(DockerAddon, "run", new=container_events_task):
resp = await api_client.post("/addons/local_ssh/restart")
assert state_changes == [AddonState.STARTUP]
assert install_addon_ssh.state == AddonState.STARTED
assert resp.status == 200
async def test_api_addon_rebuild_healthcheck(
api_client: TestClient,
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
tmp_supervisor_data,
path_extern,
):
"""Test rebuilding an addon waits for healthy."""
container.status = "running"
install_addon_ssh.path_data.mkdir()
container.attrs["Config"] = {"Healthcheck": "exists"}
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STARTUP
state_changes: list[AddonState] = []
async def container_events():
nonlocal state_changes
await install_addon_ssh.container_state_changed(
_create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.STOPPED)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
_create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.RUNNING)
)
state_changes.append(install_addon_ssh.state)
await asyncio.sleep(0)
await install_addon_ssh.container_state_changed(
_create_test_event(f"addon_{TEST_ADDON_SLUG}", ContainerState.HEALTHY)
)
async def container_events_task(*args, **kwargs):
asyncio.create_task(container_events())
with patch.object(
AddonBuild, "is_valid", new=PropertyMock(return_value=True)
), patch.object(DockerAddon, "_is_running", return_value=False), patch.object(
Addon, "need_build", new=PropertyMock(return_value=True)
), patch.object(
CpuArch, "supported", new=PropertyMock(return_value=["amd64"])
), patch.object(
DockerAddon, "run", new=container_events_task
):
resp = await api_client.post("/addons/local_ssh/rebuild")
assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
assert install_addon_ssh.state == AddonState.STARTED
assert resp.status == 200

View File

@ -1,13 +1,25 @@
"""Test Store API."""
from unittest.mock import patch
import asyncio
from unittest.mock import MagicMock, PropertyMock, patch
from aiohttp.test_utils import TestClient
import pytest
from supervisor.addons.addon import Addon
from supervisor.arch import CpuArch
from supervisor.const import AddonState
from supervisor.coresys import CoreSys
from supervisor.docker.addon import DockerAddon
from supervisor.docker.const import ContainerState
from supervisor.docker.interface import DockerInterface
from supervisor.docker.monitor import DockerContainerStateEvent
from supervisor.store.addon import AddonStore
from supervisor.store.repository import Repository
from tests.common import load_json_fixture
from tests.const import TEST_ADDON_SLUG
REPO_URL = "https://github.com/awesome-developer/awesome-repo"
@ -102,3 +114,74 @@ async def test_api_store_remove_repository(
assert response.status == 200
assert repository.source not in coresys.store.repository_urls
assert repository.slug not in coresys.store.repositories
async def test_api_store_update_healthcheck(
api_client: TestClient,
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
tmp_supervisor_data,
path_extern,
):
"""Test updating an addon with healthcheck waits for health status."""
container.status = "running"
container.attrs["Config"] = {"Healthcheck": "exists"}
install_addon_ssh.path_data.mkdir()
await install_addon_ssh.load()
with patch(
"supervisor.store.data.read_json_or_yaml_file",
return_value=load_json_fixture("addon-config-add-image.json"),
):
await coresys.store.data.update()
assert install_addon_ssh.need_update is True
state_changes: list[AddonState] = []
async def container_events():
nonlocal state_changes
await asyncio.sleep(0.01)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.STOPPED,
id="abc123",
time=1,
)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.RUNNING,
id="abc123",
time=1,
)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.HEALTHY,
id="abc123",
time=1,
)
)
async def container_events_task(*args, **kwargs):
asyncio.create_task(container_events())
with patch.object(DockerAddon, "run", new=container_events_task), patch.object(
DockerInterface, "_install"
), patch.object(DockerAddon, "_is_running", return_value=False), patch.object(
CpuArch, "supported", new=PropertyMock(return_value=["amd64"])
):
resp = await api_client.post(f"/store/addons/{TEST_ADDON_SLUG}/update")
assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
assert install_addon_ssh.state == AddonState.STARTED
assert resp.status == 200

View File

@ -1,5 +1,6 @@
"""Test BackupManager class."""
import asyncio
from shutil import rmtree
from unittest.mock import AsyncMock, MagicMock, Mock, PropertyMock, patch
@ -8,11 +9,16 @@ from dbus_fast import DBusError
import pytest
from supervisor.addons.addon import Addon
from supervisor.addons.const import AddonBackupMode
from supervisor.addons.model import AddonModel
from supervisor.backups.backup import Backup
from supervisor.backups.const import BackupType
from supervisor.backups.manager import BackupManager
from supervisor.const import FOLDER_HOMEASSISTANT, FOLDER_SHARE, CoreState
from supervisor.const import FOLDER_HOMEASSISTANT, FOLDER_SHARE, AddonState, CoreState
from supervisor.coresys import CoreSys
from supervisor.docker.addon import DockerAddon
from supervisor.docker.const import ContainerState
from supervisor.docker.monitor import DockerContainerStateEvent
from supervisor.exceptions import AddonsError, DockerError
from supervisor.homeassistant.core import HomeAssistantCore
from supervisor.homeassistant.module import HomeAssistant
@ -696,3 +702,138 @@ async def test_load_network_error(
await coresys.backups.load()
assert "Could not list backups from /data/backup_test" in caplog.text
async def test_backup_with_healthcheck(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
tmp_supervisor_data,
path_extern,
):
"""Test backup of addon with healthcheck in cold mode."""
container.status = "running"
container.attrs["Config"] = {"Healthcheck": "exists"}
install_addon_ssh.path_data.mkdir()
coresys.core.state = CoreState.RUNNING
coresys.hardware.disk.get_disk_free_space = lambda x: 5000
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STARTUP
state_changes: list[AddonState] = []
async def container_events():
nonlocal state_changes
await asyncio.sleep(0.01)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.STOPPED,
id="abc123",
time=1,
)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.RUNNING,
id="abc123",
time=1,
)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.HEALTHY,
id="abc123",
time=1,
)
)
async def container_events_task(*args, **kwargs):
asyncio.create_task(container_events())
with patch.object(DockerAddon, "run", new=container_events_task), patch.object(
AddonModel, "backup_mode", new=PropertyMock(return_value=AddonBackupMode.COLD)
), patch.object(DockerAddon, "_is_running", side_effect=[True, False, False]):
backup = await coresys.backups.do_backup_partial(
homeassistant=False, addons=["local_ssh"]
)
assert backup
assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
assert install_addon_ssh.state == AddonState.STARTED
assert coresys.core.state == CoreState.RUNNING
async def test_restore_with_healthcheck(
coresys: CoreSys,
install_addon_ssh: Addon,
container: MagicMock,
tmp_supervisor_data,
path_extern,
):
"""Test backup of addon with healthcheck in cold mode."""
container.status = "running"
container.attrs["Config"] = {"Healthcheck": "exists"}
install_addon_ssh.path_data.mkdir()
coresys.core.state = CoreState.RUNNING
coresys.hardware.disk.get_disk_free_space = lambda x: 5000
await install_addon_ssh.load()
assert install_addon_ssh.state == AddonState.STARTUP
backup = await coresys.backups.do_backup_partial(
homeassistant=False, addons=["local_ssh"]
)
state_changes: list[AddonState] = []
async def container_events():
nonlocal state_changes
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.STOPPED,
id="abc123",
time=1,
)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.RUNNING,
id="abc123",
time=1,
)
)
state_changes.append(install_addon_ssh.state)
await install_addon_ssh.container_state_changed(
DockerContainerStateEvent(
name=f"addon_{TEST_ADDON_SLUG}",
state=ContainerState.HEALTHY,
id="abc123",
time=1,
)
)
async def container_events_task(*args, **kwargs):
asyncio.create_task(container_events())
with patch.object(DockerAddon, "run", new=container_events_task), patch.object(
DockerAddon, "_is_running", return_value=False
), patch.object(AddonModel, "_validate_availability"), patch.object(
Addon, "with_ingress", new=PropertyMock(return_value=False)
):
await coresys.backups.do_restore_partial(backup, addons=["local_ssh"])
assert state_changes == [AddonState.STOPPED, AddonState.STARTUP]
assert install_addon_ssh.state == AddonState.STARTED
assert coresys.core.state == CoreState.RUNNING

View File

@ -377,6 +377,7 @@ async def tmp_supervisor_data(coresys: CoreSys, tmp_path: Path) -> Path:
coresys.config.path_audio.mkdir()
coresys.config.path_dns.mkdir()
coresys.config.path_share.mkdir()
coresys.config.path_addons_data.mkdir(parents=True)
yield tmp_path
@ -641,3 +642,15 @@ async def mount_propagation(docker: DockerAPI, coresys: CoreSys) -> None:
}
await coresys.supervisor.load()
yield
@pytest.fixture
async def container(docker: DockerAPI) -> MagicMock:
"""Mock attrs and status for container on attach."""
docker.containers.get.return_value = addon = MagicMock()
docker.containers.create.return_value = addon
docker.images.pull.return_value = addon
docker.images.build.return_value = (addon, "")
addon.status = "stopped"
addon.attrs = {"State": {"ExitCode": 0}}
yield addon

View File

@ -189,7 +189,7 @@ async def test_addon_run_docker_error(
):
"""Test docker error when addon is run."""
await coresys.dbus.timedate.connect(coresys.dbus.bus)
coresys.docker.docker.containers.create.side_effect = NotFound("Missing")
coresys.docker.containers.create.side_effect = NotFound("Missing")
docker_addon = get_docker_addon(
coresys, addonsdata_system, "basic-addon-config.json"
)

Binary file not shown.

Binary file not shown.

View File

@ -25,6 +25,10 @@ def make_mock_container_get(status: str):
return mock_container_get
async def _mock_wait_for_container() -> None:
"""Mock of wait for container."""
async def test_fixup(docker: DockerAPI, coresys: CoreSys, install_addon_ssh: Addon):
"""Test fixup rebuilds addon's container."""
docker.containers.get = make_mock_container_get("running")
@ -39,7 +43,9 @@ async def test_fixup(docker: DockerAPI, coresys: CoreSys, install_addon_ssh: Add
reference="local_ssh",
suggestions=[SuggestionType.EXECUTE_REBUILD],
)
with patch.object(Addon, "restart") as restart:
with patch.object(
Addon, "restart", return_value=_mock_wait_for_container()
) as restart:
await addon_execute_rebuild()
restart.assert_called_once()