Repair / fixup docker overlayfs issues (#1170)

* Add a repair modus

* Add repair to add-ons

* repair to cli

* Add API call

* fix sync call

* Clean all images

* Fix repair

* Fix supervisor

* Add new function to core

* fix tagging

* better style

* use retag

* new retag function

* Fix lint

* Fix import export
This commit is contained in:
Pascal Vizeli 2019-08-07 17:26:32 +02:00 committed by GitHub
parent 778bc46848
commit 2fc5e3b7d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 156 additions and 17 deletions

4
API.md
View File

@ -112,6 +112,10 @@ Output is the raw docker log.
}
```
- GET `/supervisor/repair`
Repair overlayfs issue and restore lost images
### Snapshot
- GET `/snapshots`

View File

@ -256,3 +256,38 @@ class AddonManager(CoreSysAttributes):
_LOGGER.info("Detect new Add-on after restore %s", slug)
self.local[slug] = addon
async def repair(self) -> None:
"""Repair local add-ons."""
needs_repair: List[Addon] = []
# Evaluate Add-ons to repair
for addon in self.installed:
if await addon.instance.exists():
continue
needs_repair.append(addon)
_LOGGER.info("Found %d add-ons to repair", len(needs_repair))
if not needs_repair:
return
for addon in needs_repair:
_LOGGER.info("Start repair for add-on: %s", addon.slug)
with suppress(DockerAPIError, KeyError):
# Need pull a image again
if not addon.need_build:
await addon.instance.install(addon.version, addon.image)
continue
# Need local lookup
elif addon.need_build and not addon.is_detached:
store = self.store[addon.slug]
# If this add-on is available for rebuild
if addon.version == store.version:
await addon.instance.install(addon.version, addon.image)
continue
_LOGGER.error("Can't repair %s", addon.slug)
with suppress(AddonsError):
await self.uninstall(addon.slug)

View File

@ -618,7 +618,7 @@ class Addon(AddonModel):
image_file = Path(temp, "image.tar")
if image_file.is_file():
with suppress(DockerAPIError):
await self.instance.import_image(image_file, version)
await self.instance.import_image(image_file)
else:
with suppress(DockerAPIError):
await self.instance.install(version, restore_image)

View File

@ -130,6 +130,7 @@ class RestAPI(CoreSysAttributes):
web.post("/supervisor/update", api_supervisor.update),
web.post("/supervisor/reload", api_supervisor.reload),
web.post("/supervisor/options", api_supervisor.options),
web.post("/supervisor/repair", api_supervisor.repair),
]
)

View File

@ -161,6 +161,11 @@ class APISupervisor(CoreSysAttributes):
"""Reload add-ons, configuration, etc."""
return asyncio.shield(self.sys_updater.reload())
@api_process
def repair(self, request: web.Request) -> Awaitable[None]:
"""Try to repair the local setup / overlayfs."""
return asyncio.shield(self.sys_core.repair())
@api_process_raw(CONTENT_TYPE_BINARY)
def logs(self, request: web.Request) -> Awaitable[bytes]:
"""Return supervisor Docker logs."""

View File

@ -170,3 +170,18 @@ class HassIO(CoreSysAttributes):
"""Update last boot time."""
self.sys_config.last_boot = self.sys_hardware.last_boot
self.sys_config.save_data()
async def repair(self):
"""Repair system integrity."""
await self.sys_run_in_executor(self.sys_docker.repair)
# Restore core functionality
await self.sys_addons.repair()
await self.sys_homeassistant.repair()
# Fix HassOS specific
if self.sys_hassos.available:
await self.sys_hassos.repair_cli()
# Tag version for latest
await self.sys_supervisor.repair()

View File

@ -139,3 +139,34 @@ class DockerAPI:
container.remove(force=True)
return CommandReturn(result.get("StatusCode"), output)
def repair(self) -> None:
"""Repair local docker overlayfs2 issues."""
_LOGGER.info("Prune stale containers")
try:
output = self.docker.api.prune_containers()
_LOGGER.debug("Containers prune: %s", output)
except docker.errors.APIError as err:
_LOGGER.warning("Error for containers prune: %s", err)
_LOGGER.info("Prune stale images")
try:
output = self.docker.api.prune_images(filters={"dangling": False})
_LOGGER.debug("Images prune: %s", output)
except docker.errors.APIError as err:
_LOGGER.warning("Error for images prune: %s", err)
_LOGGER.info("Prune stale builds")
try:
output = self.docker.api.prune_builds()
_LOGGER.debug("Builds prune: %s", output)
except docker.errors.APIError as err:
_LOGGER.warning("Error for builds prune: %s", err)
_LOGGER.info("Prune stale volumes")
try:
output = self.docker.api.prune_builds()
_LOGGER.debug("Volumes prune: %s", output)
except docker.errors.APIError as err:
_LOGGER.warning("Error for volumes prune: %s", err)

View File

@ -397,7 +397,7 @@ class DockerAddon(DockerInterface):
Need run inside executor.
"""
try:
image = self.sys_docker.api.get_image(self.image)
image = self.sys_docker.api.get_image(f"{self.image}:{self.version}")
except docker.errors.DockerException as err:
_LOGGER.error("Can't fetch image %s: %s", self.image, err)
raise DockerAPIError() from None
@ -414,11 +414,11 @@ class DockerAddon(DockerInterface):
_LOGGER.info("Export image %s done", self.image)
@process_lock
def import_image(self, tar_file: Path, tag: str) -> Awaitable[None]:
def import_image(self, tar_file: Path) -> Awaitable[None]:
"""Import a tar file as image."""
return self.sys_run_in_executor(self._import_image, tar_file, tag)
return self.sys_run_in_executor(self._import_image, tar_file)
def _import_image(self, tar_file: Path, tag: str) -> None:
def _import_image(self, tar_file: Path) -> None:
"""Import a tar file as image.
Need run inside executor.
@ -427,14 +427,13 @@ class DockerAddon(DockerInterface):
with tar_file.open("rb") as read_tar:
self.sys_docker.api.load_image(read_tar, quiet=True)
docker_image = self.sys_docker.images.get(self.image)
docker_image.tag(self.image, tag=tag)
docker_image = self.sys_docker.images.get(f"{self.image}:{self.version}")
except (docker.errors.DockerException, OSError) as err:
_LOGGER.error("Can't import image %s: %s", self.image, err)
raise DockerAPIError() from None
_LOGGER.info("Import image %s and tag %s", tar_file, tag)
self._meta = docker_image.attrs
_LOGGER.info("Import image %s and version %s", tar_file, self.version)
with suppress(DockerAPIError):
self._cleanup()

View File

@ -103,13 +103,10 @@ class DockerInterface(CoreSysAttributes):
Need run inside executor.
"""
try:
docker_image = self.sys_docker.images.get(self.image)
assert f"{self.image}:{self.version}" in docker_image.tags
except (docker.errors.DockerException, AssertionError):
return False
return True
with suppress(docker.errors.DockerException):
self.sys_docker.images.get(f"{self.image}:{self.version}")
return True
return False
def is_running(self) -> Awaitable[bool]:
"""Return True if Docker is running.

View File

@ -2,6 +2,7 @@
from ipaddress import IPv4Address
import logging
import os
from typing import Awaitable
import docker
@ -49,3 +50,21 @@ class DockerSupervisor(DockerInterface, CoreSysAttributes):
self.sys_docker.network.attach_container(
docker_container, alias=["hassio"], ipv4=self.sys_docker.network.supervisor
)
def retag(self) -> Awaitable[None]:
"""Retag latest image to version."""
return self.sys_run_in_executor(self._retag)
def _retag(self) -> None:
"""Retag latest image to version.
Need run inside executor.
"""
try:
docker_container = self.sys_docker.containers.get(self.name)
docker_container.image.tag(self.image, tag=self.version)
docker_container.image.tag(self.image, tag="latest")
except docker.errors.DockerException as err:
_LOGGER.error("Can't retag supervisor version: %s", err)
raise DockerAPIError() from None

View File

@ -195,3 +195,14 @@ class HassOS(CoreSysAttributes):
except DockerAPIError:
_LOGGER.error("HassOS CLI update fails")
raise HassOSUpdateError() from None
async def repair_cli(self) -> None:
"""Repair CLI container."""
if await self.instance.exists():
return
_LOGGER.info("Repair HassOS CLI %s", self.version_cli)
try:
await self.instance.install(self.version_cli, latest=True)
except DockerAPIError:
_LOGGER.error("Repairing of HassOS CLI fails")

View File

@ -597,3 +597,14 @@ class HomeAssistant(JsonConfig, CoreSysAttributes):
self._error_state = True
raise HomeAssistantError()
async def repair(self):
"""Repair local Home Assistant data."""
if await self.instance.exists():
return
_LOGGER.info("Repair Home Assistant %s", self.version)
try:
await self.instance.install(self.version)
except DockerAPIError:
_LOGGER.error("Repairing of Home Assistant fails")

View File

@ -9,7 +9,7 @@ from typing import Awaitable, Optional
import aiohttp
from .const import URL_HASSIO_APPARMOR
from .const import URL_HASSIO_APPARMOR, HASSIO_VERSION
from .coresys import CoreSys, CoreSysAttributes
from .docker.stats import DockerStats
from .docker.supervisor import DockerSupervisor
@ -54,7 +54,7 @@ class Supervisor(CoreSysAttributes):
@property
def version(self) -> str:
"""Return version of running Home Assistant."""
return self.instance.version
return HASSIO_VERSION
@property
def latest_version(self) -> str:
@ -136,3 +136,14 @@ class Supervisor(CoreSysAttributes):
return await self.instance.stats()
except DockerAPIError:
raise SupervisorError() from None
async def repair(self):
"""Repair local Supervisor data."""
if await self.instance.exists():
return
_LOGGER.info("Repair Supervisor %s", self.version)
try:
await self.instance.retag()
except DockerAPIError:
_LOGGER.error("Repairing of Supervisor fails")