Ensure MariaDB/MySQL can be purged and handle states being deleted out from under the recorder (#43610)

* MariaDB doesn't purge #42402

This addresses  home-assistant#42402
Relationships within table "states" and between tables "states" and "events " home-assistant#40467 prevent the purge from working correctly. The database increases w/o any purge.
This proposal sets related indices to NULL and permits deleting of rows.
Further explanations can be found here home-assistant#42402
This proposal also allows to purge the tables "events" and "states" in any order.

* Update models.py

Corrected for Black style requirements

* Update homeassistant/components/recorder/models.py

Co-authored-by: J. Nick Koston <nick@koston.org>

* Add the options to foreign key constraints

* purge old states when database gets deleted out from under us

* pylint

Co-authored-by: J. Nick Koston <nick@koston.org>

homeassistant/components/recorder/__init__.py

@@ -514,6 +514,14 @@ class Recorder(threading.Thread):
                         self.event_session.expunge(dbstate)
                 self._pending_expunge = []
             self.event_session.commit()
+        except exc.IntegrityError as err:
+            _LOGGER.error(
+                "Integrity error executing query (database likely deleted out from under us): %s",
+                err,
+            )
+            self.event_session.rollback()
+            self._old_states = {}
+            raise
         except Exception as err:
             _LOGGER.error("Error executing query: %s", err)
             self.event_session.rollback()

homeassistant/components/recorder/migration.py

@@ -1,12 +1,13 @@
 """Schema migration helpers."""
 import logging
 
-from sqlalchemy import Table, text
+from sqlalchemy import ForeignKeyConstraint, MetaData, Table, text
 from sqlalchemy.engine import reflection
 from sqlalchemy.exc import InternalError, OperationalError, SQLAlchemyError
+from sqlalchemy.schema import AddConstraint, DropConstraint
 
 from .const import DOMAIN
-from .models import SCHEMA_VERSION, Base, SchemaChanges
+from .models import SCHEMA_VERSION, TABLE_STATES, Base, SchemaChanges
 from .util import session_scope
 
 _LOGGER = logging.getLogger(__name__)
@@ -205,6 +206,39 @@ def _add_columns(engine, table_name, columns_def):
             )
 
 
+def _update_states_table_with_foreign_key_options(engine):
+    """Add the options to foreign key constraints."""
+    inspector = reflection.Inspector.from_engine(engine)
+    alters = []
+    for foreign_key in inspector.get_foreign_keys(TABLE_STATES):
+        if foreign_key["name"] and not foreign_key["options"]:
+            alters.append(
+                {
+                    "old_fk": ForeignKeyConstraint((), (), name=foreign_key["name"]),
+                    "columns": foreign_key["constrained_columns"],
+                }
+            )
+
+    if not alters:
+        return
+
+    states_key_constraints = Base.metadata.tables[TABLE_STATES].foreign_key_constraints
+    old_states_table = Table(  # noqa: F841 pylint: disable=unused-variable
+        TABLE_STATES, MetaData(), *[alter["old_fk"] for alter in alters]
+    )
+
+    for alter in alters:
+        try:
+            engine.execute(DropConstraint(alter["old_fk"]))
+            for fkc in states_key_constraints:
+                if fkc.column_keys == alter["columns"]:
+                    engine.execute(AddConstraint(fkc))
+        except (InternalError, OperationalError):
+            _LOGGER.exception(
+                "Could not update foreign options in %s table", TABLE_STATES
+            )
+
+
 def _apply_update(engine, new_version, old_version):
     """Perform operations to bring schema up to date."""
     if new_version == 1:
@@ -277,6 +311,8 @@ def _apply_update(engine, new_version, old_version):
         _drop_index(engine, "states", "ix_states_entity_id")
         _create_index(engine, "events", "ix_events_event_type_time_fired")
         _drop_index(engine, "events", "ix_events_event_type")
+    elif new_version == 10:
+        _update_states_table_with_foreign_key_options(engine)
     else:
         raise ValueError(f"No schema migration defined for version {new_version}")
 

homeassistant/components/recorder/models.py

@@ -25,7 +25,7 @@ import homeassistant.util.dt as dt_util
 # pylint: disable=invalid-name
 Base = declarative_base()
 
-SCHEMA_VERSION = 9
+SCHEMA_VERSION = 10
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -36,7 +36,7 @@ TABLE_STATES = "states"
 TABLE_RECORDER_RUNS = "recorder_runs"
 TABLE_SCHEMA_CHANGES = "schema_changes"
 
-ALL_TABLES = [TABLE_EVENTS, TABLE_STATES, TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES]
+ALL_TABLES = [TABLE_STATES, TABLE_EVENTS, TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES]
 
 
 class Events(Base):  # type: ignore
@@ -102,11 +102,15 @@ class States(Base):  # type: ignore
     entity_id = Column(String(255))
     state = Column(String(255))
     attributes = Column(Text)
-    event_id = Column(Integer, ForeignKey("events.event_id"), index=True)
+    event_id = Column(
+        Integer, ForeignKey("events.event_id", ondelete="CASCADE"), index=True
+    )
     last_changed = Column(DateTime(timezone=True), default=dt_util.utcnow)
     last_updated = Column(DateTime(timezone=True), default=dt_util.utcnow, index=True)
     created = Column(DateTime(timezone=True), default=dt_util.utcnow)
-    old_state_id = Column(Integer, ForeignKey("states.state_id"))
+    old_state_id = Column(
+        Integer, ForeignKey("states.state_id", ondelete="SET NULL"), index=True
+    )
     event = relationship("Events", uselist=False)
     old_state = relationship("States", remote_side=[state_id])