Repair / fixup docker overlayfs issues (#1170)
* Add a repair modus * Add repair to add-ons * repair to cli * Add API call * fix sync call * Clean all images * Fix repair * Fix supervisor * Add new function to core * fix tagging * better style * use retag * new retag function * Fix lint * Fix import export
This commit is contained in:
parent
778bc46848
commit
2fc5e3b7d9
4
API.md
4
API.md
|
@ -112,6 +112,10 @@ Output is the raw docker log.
|
|||
}
|
||||
```
|
||||
|
||||
- GET `/supervisor/repair`
|
||||
|
||||
Repair overlayfs issue and restore lost images
|
||||
|
||||
### Snapshot
|
||||
|
||||
- GET `/snapshots`
|
||||
|
|
|
@ -256,3 +256,38 @@ class AddonManager(CoreSysAttributes):
|
|||
|
||||
_LOGGER.info("Detect new Add-on after restore %s", slug)
|
||||
self.local[slug] = addon
|
||||
|
||||
async def repair(self) -> None:
|
||||
"""Repair local add-ons."""
|
||||
needs_repair: List[Addon] = []
|
||||
|
||||
# Evaluate Add-ons to repair
|
||||
for addon in self.installed:
|
||||
if await addon.instance.exists():
|
||||
continue
|
||||
needs_repair.append(addon)
|
||||
|
||||
_LOGGER.info("Found %d add-ons to repair", len(needs_repair))
|
||||
if not needs_repair:
|
||||
return
|
||||
|
||||
for addon in needs_repair:
|
||||
_LOGGER.info("Start repair for add-on: %s", addon.slug)
|
||||
|
||||
with suppress(DockerAPIError, KeyError):
|
||||
# Need pull a image again
|
||||
if not addon.need_build:
|
||||
await addon.instance.install(addon.version, addon.image)
|
||||
continue
|
||||
|
||||
# Need local lookup
|
||||
elif addon.need_build and not addon.is_detached:
|
||||
store = self.store[addon.slug]
|
||||
# If this add-on is available for rebuild
|
||||
if addon.version == store.version:
|
||||
await addon.instance.install(addon.version, addon.image)
|
||||
continue
|
||||
|
||||
_LOGGER.error("Can't repair %s", addon.slug)
|
||||
with suppress(AddonsError):
|
||||
await self.uninstall(addon.slug)
|
||||
|
|
|
@ -618,7 +618,7 @@ class Addon(AddonModel):
|
|||
image_file = Path(temp, "image.tar")
|
||||
if image_file.is_file():
|
||||
with suppress(DockerAPIError):
|
||||
await self.instance.import_image(image_file, version)
|
||||
await self.instance.import_image(image_file)
|
||||
else:
|
||||
with suppress(DockerAPIError):
|
||||
await self.instance.install(version, restore_image)
|
||||
|
|
|
@ -130,6 +130,7 @@ class RestAPI(CoreSysAttributes):
|
|||
web.post("/supervisor/update", api_supervisor.update),
|
||||
web.post("/supervisor/reload", api_supervisor.reload),
|
||||
web.post("/supervisor/options", api_supervisor.options),
|
||||
web.post("/supervisor/repair", api_supervisor.repair),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
|
@ -161,6 +161,11 @@ class APISupervisor(CoreSysAttributes):
|
|||
"""Reload add-ons, configuration, etc."""
|
||||
return asyncio.shield(self.sys_updater.reload())
|
||||
|
||||
@api_process
|
||||
def repair(self, request: web.Request) -> Awaitable[None]:
|
||||
"""Try to repair the local setup / overlayfs."""
|
||||
return asyncio.shield(self.sys_core.repair())
|
||||
|
||||
@api_process_raw(CONTENT_TYPE_BINARY)
|
||||
def logs(self, request: web.Request) -> Awaitable[bytes]:
|
||||
"""Return supervisor Docker logs."""
|
||||
|
|
|
@ -170,3 +170,18 @@ class HassIO(CoreSysAttributes):
|
|||
"""Update last boot time."""
|
||||
self.sys_config.last_boot = self.sys_hardware.last_boot
|
||||
self.sys_config.save_data()
|
||||
|
||||
async def repair(self):
|
||||
"""Repair system integrity."""
|
||||
await self.sys_run_in_executor(self.sys_docker.repair)
|
||||
|
||||
# Restore core functionality
|
||||
await self.sys_addons.repair()
|
||||
await self.sys_homeassistant.repair()
|
||||
|
||||
# Fix HassOS specific
|
||||
if self.sys_hassos.available:
|
||||
await self.sys_hassos.repair_cli()
|
||||
|
||||
# Tag version for latest
|
||||
await self.sys_supervisor.repair()
|
||||
|
|
|
@ -139,3 +139,34 @@ class DockerAPI:
|
|||
container.remove(force=True)
|
||||
|
||||
return CommandReturn(result.get("StatusCode"), output)
|
||||
|
||||
def repair(self) -> None:
|
||||
"""Repair local docker overlayfs2 issues."""
|
||||
|
||||
_LOGGER.info("Prune stale containers")
|
||||
try:
|
||||
output = self.docker.api.prune_containers()
|
||||
_LOGGER.debug("Containers prune: %s", output)
|
||||
except docker.errors.APIError as err:
|
||||
_LOGGER.warning("Error for containers prune: %s", err)
|
||||
|
||||
_LOGGER.info("Prune stale images")
|
||||
try:
|
||||
output = self.docker.api.prune_images(filters={"dangling": False})
|
||||
_LOGGER.debug("Images prune: %s", output)
|
||||
except docker.errors.APIError as err:
|
||||
_LOGGER.warning("Error for images prune: %s", err)
|
||||
|
||||
_LOGGER.info("Prune stale builds")
|
||||
try:
|
||||
output = self.docker.api.prune_builds()
|
||||
_LOGGER.debug("Builds prune: %s", output)
|
||||
except docker.errors.APIError as err:
|
||||
_LOGGER.warning("Error for builds prune: %s", err)
|
||||
|
||||
_LOGGER.info("Prune stale volumes")
|
||||
try:
|
||||
output = self.docker.api.prune_builds()
|
||||
_LOGGER.debug("Volumes prune: %s", output)
|
||||
except docker.errors.APIError as err:
|
||||
_LOGGER.warning("Error for volumes prune: %s", err)
|
||||
|
|
|
@ -397,7 +397,7 @@ class DockerAddon(DockerInterface):
|
|||
Need run inside executor.
|
||||
"""
|
||||
try:
|
||||
image = self.sys_docker.api.get_image(self.image)
|
||||
image = self.sys_docker.api.get_image(f"{self.image}:{self.version}")
|
||||
except docker.errors.DockerException as err:
|
||||
_LOGGER.error("Can't fetch image %s: %s", self.image, err)
|
||||
raise DockerAPIError() from None
|
||||
|
@ -414,11 +414,11 @@ class DockerAddon(DockerInterface):
|
|||
_LOGGER.info("Export image %s done", self.image)
|
||||
|
||||
@process_lock
|
||||
def import_image(self, tar_file: Path, tag: str) -> Awaitable[None]:
|
||||
def import_image(self, tar_file: Path) -> Awaitable[None]:
|
||||
"""Import a tar file as image."""
|
||||
return self.sys_run_in_executor(self._import_image, tar_file, tag)
|
||||
return self.sys_run_in_executor(self._import_image, tar_file)
|
||||
|
||||
def _import_image(self, tar_file: Path, tag: str) -> None:
|
||||
def _import_image(self, tar_file: Path) -> None:
|
||||
"""Import a tar file as image.
|
||||
|
||||
Need run inside executor.
|
||||
|
@ -427,14 +427,13 @@ class DockerAddon(DockerInterface):
|
|||
with tar_file.open("rb") as read_tar:
|
||||
self.sys_docker.api.load_image(read_tar, quiet=True)
|
||||
|
||||
docker_image = self.sys_docker.images.get(self.image)
|
||||
docker_image.tag(self.image, tag=tag)
|
||||
docker_image = self.sys_docker.images.get(f"{self.image}:{self.version}")
|
||||
except (docker.errors.DockerException, OSError) as err:
|
||||
_LOGGER.error("Can't import image %s: %s", self.image, err)
|
||||
raise DockerAPIError() from None
|
||||
|
||||
_LOGGER.info("Import image %s and tag %s", tar_file, tag)
|
||||
self._meta = docker_image.attrs
|
||||
_LOGGER.info("Import image %s and version %s", tar_file, self.version)
|
||||
|
||||
with suppress(DockerAPIError):
|
||||
self._cleanup()
|
||||
|
|
|
@ -103,13 +103,10 @@ class DockerInterface(CoreSysAttributes):
|
|||
|
||||
Need run inside executor.
|
||||
"""
|
||||
try:
|
||||
docker_image = self.sys_docker.images.get(self.image)
|
||||
assert f"{self.image}:{self.version}" in docker_image.tags
|
||||
except (docker.errors.DockerException, AssertionError):
|
||||
return False
|
||||
|
||||
return True
|
||||
with suppress(docker.errors.DockerException):
|
||||
self.sys_docker.images.get(f"{self.image}:{self.version}")
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_running(self) -> Awaitable[bool]:
|
||||
"""Return True if Docker is running.
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
from ipaddress import IPv4Address
|
||||
import logging
|
||||
import os
|
||||
from typing import Awaitable
|
||||
|
||||
import docker
|
||||
|
||||
|
@ -49,3 +50,21 @@ class DockerSupervisor(DockerInterface, CoreSysAttributes):
|
|||
self.sys_docker.network.attach_container(
|
||||
docker_container, alias=["hassio"], ipv4=self.sys_docker.network.supervisor
|
||||
)
|
||||
|
||||
def retag(self) -> Awaitable[None]:
|
||||
"""Retag latest image to version."""
|
||||
return self.sys_run_in_executor(self._retag)
|
||||
|
||||
def _retag(self) -> None:
|
||||
"""Retag latest image to version.
|
||||
|
||||
Need run inside executor.
|
||||
"""
|
||||
try:
|
||||
docker_container = self.sys_docker.containers.get(self.name)
|
||||
|
||||
docker_container.image.tag(self.image, tag=self.version)
|
||||
docker_container.image.tag(self.image, tag="latest")
|
||||
except docker.errors.DockerException as err:
|
||||
_LOGGER.error("Can't retag supervisor version: %s", err)
|
||||
raise DockerAPIError() from None
|
||||
|
|
|
@ -195,3 +195,14 @@ class HassOS(CoreSysAttributes):
|
|||
except DockerAPIError:
|
||||
_LOGGER.error("HassOS CLI update fails")
|
||||
raise HassOSUpdateError() from None
|
||||
|
||||
async def repair_cli(self) -> None:
|
||||
"""Repair CLI container."""
|
||||
if await self.instance.exists():
|
||||
return
|
||||
|
||||
_LOGGER.info("Repair HassOS CLI %s", self.version_cli)
|
||||
try:
|
||||
await self.instance.install(self.version_cli, latest=True)
|
||||
except DockerAPIError:
|
||||
_LOGGER.error("Repairing of HassOS CLI fails")
|
||||
|
|
|
@ -597,3 +597,14 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
|
|||
|
||||
self._error_state = True
|
||||
raise HomeAssistantError()
|
||||
|
||||
async def repair(self):
|
||||
"""Repair local Home Assistant data."""
|
||||
if await self.instance.exists():
|
||||
return
|
||||
|
||||
_LOGGER.info("Repair Home Assistant %s", self.version)
|
||||
try:
|
||||
await self.instance.install(self.version)
|
||||
except DockerAPIError:
|
||||
_LOGGER.error("Repairing of Home Assistant fails")
|
||||
|
|
|
@ -9,7 +9,7 @@ from typing import Awaitable, Optional
|
|||
|
||||
import aiohttp
|
||||
|
||||
from .const import URL_HASSIO_APPARMOR
|
||||
from .const import URL_HASSIO_APPARMOR, HASSIO_VERSION
|
||||
from .coresys import CoreSys, CoreSysAttributes
|
||||
from .docker.stats import DockerStats
|
||||
from .docker.supervisor import DockerSupervisor
|
||||
|
@ -54,7 +54,7 @@ class Supervisor(CoreSysAttributes):
|
|||
@property
|
||||
def version(self) -> str:
|
||||
"""Return version of running Home Assistant."""
|
||||
return self.instance.version
|
||||
return HASSIO_VERSION
|
||||
|
||||
@property
|
||||
def latest_version(self) -> str:
|
||||
|
@ -136,3 +136,14 @@ class Supervisor(CoreSysAttributes):
|
|||
return await self.instance.stats()
|
||||
except DockerAPIError:
|
||||
raise SupervisorError() from None
|
||||
|
||||
async def repair(self):
|
||||
"""Repair local Supervisor data."""
|
||||
if await self.instance.exists():
|
||||
return
|
||||
|
||||
_LOGGER.info("Repair Supervisor %s", self.version)
|
||||
try:
|
||||
await self.instance.retag()
|
||||
except DockerAPIError:
|
||||
_LOGGER.error("Repairing of Supervisor fails")
|
||||
|
|
Loading…
Reference in New Issue