"""Test data purging."""

from datetime import datetime, timedelta
import json
import sqlite3
from unittest.mock import patch

from freezegun import freeze_time
import pytest
from sqlalchemy.exc import DatabaseError, OperationalError
from sqlalchemy.orm.session import Session

from homeassistant.components import recorder
from homeassistant.components.recorder.const import SupportedDialect
from homeassistant.components.recorder.db_schema import (
    Events,
    EventTypes,
    RecorderRuns,
    StateAttributes,
    States,
    StatesMeta,
    StatisticsRuns,
    StatisticsShortTerm,
)
from homeassistant.components.recorder.history import get_significant_states
from homeassistant.components.recorder.purge import purge_old_data
from homeassistant.components.recorder.queries import select_event_type_ids
from homeassistant.components.recorder.services import (
    SERVICE_PURGE,
    SERVICE_PURGE_ENTITIES,
)
from homeassistant.components.recorder.tasks import PurgeTask
from homeassistant.components.recorder.util import session_scope
from homeassistant.const import EVENT_STATE_CHANGED, EVENT_THEMES_UPDATED, STATE_ON
from homeassistant.core import HomeAssistant
from homeassistant.helpers.typing import ConfigType
from homeassistant.util import dt as dt_util

from .common import (
    async_recorder_block_till_done,
    async_wait_purge_done,
    async_wait_recording_done,
    convert_pending_events_to_event_types,
    convert_pending_states_to_meta,
)

from tests.typing import RecorderInstanceGenerator

TEST_EVENT_TYPES = (
    "EVENT_TEST_AUTOPURGE",
    "EVENT_TEST_PURGE",
    "EVENT_TEST",
    "EVENT_TEST_AUTOPURGE_WITH_EVENT_DATA",
    "EVENT_TEST_PURGE_WITH_EVENT_DATA",
    "EVENT_TEST_WITH_EVENT_DATA",
)


@pytest.fixture(name="use_sqlite")
def mock_use_sqlite(request):
    """Pytest fixture to switch purge method."""
    with patch(
        "homeassistant.components.recorder.core.Recorder.dialect_name",
        return_value=SupportedDialect.SQLITE
        if request.param
        else SupportedDialect.MYSQL,
    ):
        yield


async def test_purge_big_database(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting 2/3 old states from a big database."""

    instance = await async_setup_recorder_instance(hass)

    for _ in range(12):
        await _add_test_states(hass, wait_recording_done=False)
    await async_wait_recording_done(hass)

    with (
        patch.object(instance, "max_bind_vars", 72),
        patch.object(instance.database_engine, "max_bind_vars", 72),
        session_scope(hass=hass) as session,
    ):
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.count() == 72
        assert state_attributes.count() == 3

        purge_before = dt_util.utcnow() - timedelta(days=4)

        finished = purge_old_data(
            instance,
            purge_before,
            states_batch_size=1,
            events_batch_size=1,
            repack=False,
        )
        assert not finished
        assert states.count() == 24
        assert state_attributes.count() == 1


async def test_purge_old_states(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting old states."""
    instance = await async_setup_recorder_instance(hass)

    await _add_test_states(hass)

    # make sure we start with 6 states
    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)

        assert states.count() == 6
        assert states[0].old_state_id is None
        assert states[5].old_state_id == states[4].state_id
        assert state_attributes.count() == 3

        events = session.query(Events).filter(Events.event_type == "state_changed")
        assert events.count() == 0
        assert "test.recorder2" in instance.states_manager._last_committed_id

        purge_before = dt_util.utcnow() - timedelta(days=4)

        # run purge_old_data()
        finished = purge_old_data(
            instance,
            purge_before,
            states_batch_size=1,
            events_batch_size=1,
            repack=False,
        )
        assert not finished
        assert states.count() == 2
        assert state_attributes.count() == 1

        assert "test.recorder2" in instance.states_manager._last_committed_id

        states_after_purge = list(session.query(States))
        # Since these states are deleted in batches, we can't guarantee the order
        # but we can look them up by state
        state_map_by_state = {state.state: state for state in states_after_purge}
        dontpurgeme_5 = state_map_by_state["dontpurgeme_5"]
        dontpurgeme_4 = state_map_by_state["dontpurgeme_4"]

        assert dontpurgeme_5.old_state_id == dontpurgeme_4.state_id
        assert dontpurgeme_4.old_state_id is None

        finished = purge_old_data(instance, purge_before, repack=False)
        assert finished
        assert states.count() == 2
        assert state_attributes.count() == 1

        assert "test.recorder2" in instance.states_manager._last_committed_id

        # run purge_old_data again
        purge_before = dt_util.utcnow()
        finished = purge_old_data(
            instance,
            purge_before,
            states_batch_size=1,
            events_batch_size=1,
            repack=False,
        )
        assert not finished
        assert states.count() == 0
        assert state_attributes.count() == 0

        assert "test.recorder2" not in instance.states_manager._last_committed_id

    # Add some more states
    await _add_test_states(hass)

    # make sure we start with 6 states
    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 6
        assert states[0].old_state_id is None
        assert states[5].old_state_id == states[4].state_id

        events = session.query(Events).filter(Events.event_type == "state_changed")
        assert events.count() == 0
        assert "test.recorder2" in instance.states_manager._last_committed_id

        state_attributes = session.query(StateAttributes)
        assert state_attributes.count() == 3


async def test_purge_old_states_encouters_database_corruption(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    recorder_db_url: str,
) -> None:
    """Test database image image is malformed while deleting old states."""
    if recorder_db_url.startswith(("mysql://", "postgresql://")):
        # This test is specific for SQLite, wiping the database on error only happens
        # with SQLite.
        return

    await async_setup_recorder_instance(hass)

    await _add_test_states(hass)
    await async_wait_recording_done(hass)

    sqlite3_exception = DatabaseError("statement", {}, [])
    sqlite3_exception.__cause__ = sqlite3.DatabaseError()

    with (
        patch(
            "homeassistant.components.recorder.core.move_away_broken_database"
        ) as move_away,
        patch(
            "homeassistant.components.recorder.purge.purge_old_data",
            side_effect=sqlite3_exception,
        ),
    ):
        await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, {"keep_days": 0})
        await hass.async_block_till_done()
        await async_wait_recording_done(hass)

    assert move_away.called

    # Ensure the whole database was reset due to the database error
    with session_scope(hass=hass) as session:
        states_after_purge = session.query(States)
        assert states_after_purge.count() == 0


async def test_purge_old_states_encounters_temporary_mysql_error(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    caplog: pytest.LogCaptureFixture,
) -> None:
    """Test retry on specific mysql operational errors."""
    instance = await async_setup_recorder_instance(hass)

    await _add_test_states(hass)
    await async_wait_recording_done(hass)

    mysql_exception = OperationalError("statement", {}, [])
    mysql_exception.orig = Exception(1205, "retryable")

    with (
        patch("homeassistant.components.recorder.util.time.sleep") as sleep_mock,
        patch(
            "homeassistant.components.recorder.purge._purge_old_recorder_runs",
            side_effect=[mysql_exception, None],
        ),
        patch.object(instance.engine.dialect, "name", "mysql"),
    ):
        await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, {"keep_days": 0})
        await hass.async_block_till_done()
        await async_wait_recording_done(hass)
        await async_wait_recording_done(hass)

    assert "retrying" in caplog.text
    assert sleep_mock.called


async def test_purge_old_states_encounters_operational_error(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    caplog: pytest.LogCaptureFixture,
) -> None:
    """Test error on operational errors that are not mysql does not retry."""
    await async_setup_recorder_instance(hass)

    await _add_test_states(hass)
    await async_wait_recording_done(hass)

    exception = OperationalError("statement", {}, [])

    with patch(
        "homeassistant.components.recorder.purge._purge_old_recorder_runs",
        side_effect=exception,
    ):
        await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, {"keep_days": 0})
        await hass.async_block_till_done()
        await async_wait_recording_done(hass)
        await async_wait_recording_done(hass)

    assert "retrying" not in caplog.text
    assert "Error executing purge" in caplog.text


async def test_purge_old_events(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting old events."""
    instance = await async_setup_recorder_instance(hass)

    await _add_test_events(hass)

    with session_scope(hass=hass) as session:
        events = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(TEST_EVENT_TYPES))
        )
        assert events.count() == 6

        purge_before = dt_util.utcnow() - timedelta(days=4)

        # run purge_old_data()
        finished = purge_old_data(
            instance,
            purge_before,
            repack=False,
            events_batch_size=1,
            states_batch_size=1,
        )
        assert not finished
        all_events = events.all()
        assert events.count() == 2, f"Should have 2 events left: {all_events}"

        # we should only have 2 events left
        finished = purge_old_data(
            instance,
            purge_before,
            repack=False,
            events_batch_size=1,
            states_batch_size=1,
        )
        assert finished
        assert events.count() == 2


async def test_purge_old_recorder_runs(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting old recorder runs keeps current run."""
    instance = await async_setup_recorder_instance(hass)

    await _add_test_recorder_runs(hass)

    # make sure we start with 7 recorder runs
    with session_scope(hass=hass) as session:
        recorder_runs = session.query(RecorderRuns)
        assert recorder_runs.count() == 7

        purge_before = dt_util.utcnow()

        # run purge_old_data()
        finished = purge_old_data(
            instance,
            purge_before,
            repack=False,
            events_batch_size=1,
            states_batch_size=1,
        )
        assert not finished

        finished = purge_old_data(
            instance,
            purge_before,
            repack=False,
            events_batch_size=1,
            states_batch_size=1,
        )
        assert finished
        assert recorder_runs.count() == 1


async def test_purge_old_statistics_runs(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting old statistics runs keeps the latest run."""
    instance = await async_setup_recorder_instance(hass)

    await _add_test_statistics_runs(hass)

    # make sure we start with 7 statistics runs
    with session_scope(hass=hass) as session:
        statistics_runs = session.query(StatisticsRuns)
        assert statistics_runs.count() == 7

        purge_before = dt_util.utcnow()

        # run purge_old_data()
        finished = purge_old_data(instance, purge_before, repack=False)
        assert not finished

        finished = purge_old_data(instance, purge_before, repack=False)
        assert finished
        assert statistics_runs.count() == 1


@pytest.mark.parametrize("use_sqlite", [True, False], indirect=True)
async def test_purge_method(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    caplog: pytest.LogCaptureFixture,
    use_sqlite: bool,
) -> None:
    """Test purge method."""

    def assert_recorder_runs_equal(run1, run2):
        assert run1.run_id == run2.run_id
        assert run1.start == run2.start
        assert run1.end == run2.end
        assert run1.closed_incorrect == run2.closed_incorrect
        assert run1.created == run2.created

    def assert_statistic_runs_equal(run1, run2):
        assert run1.run_id == run2.run_id
        assert run1.start == run2.start

    await async_setup_recorder_instance(hass)

    service_data = {"keep_days": 4}
    await _add_test_events(hass)
    await _add_test_states(hass)
    await _add_test_statistics(hass)
    await _add_test_recorder_runs(hass)
    await _add_test_statistics_runs(hass)
    await hass.async_block_till_done()
    await async_wait_recording_done(hass)

    # make sure we start with 6 states
    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 6

        events = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(TEST_EVENT_TYPES))
        )
        assert events.count() == 6

        statistics = session.query(StatisticsShortTerm)
        assert statistics.count() == 6

        recorder_runs = session.query(RecorderRuns)
        assert recorder_runs.count() == 7
        runs_before_purge = recorder_runs.all()

        statistics_runs = session.query(StatisticsRuns).order_by(StatisticsRuns.run_id)
        assert statistics_runs.count() == 7
        statistic_runs_before_purge = statistics_runs.all()

        for itm in runs_before_purge:
            session.expunge(itm)
        for itm in statistic_runs_before_purge:
            session.expunge(itm)

    await hass.async_block_till_done()
    await async_wait_purge_done(hass)

    # run purge method - no service data, use defaults
    await hass.services.async_call("recorder", "purge")
    await hass.async_block_till_done()

    # Small wait for recorder thread
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        events = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(TEST_EVENT_TYPES))
        )
        statistics = session.query(StatisticsShortTerm)

        # only purged old states, events and statistics
        assert states.count() == 4
        assert events.count() == 4
        assert statistics.count() == 4

    # run purge method - correct service data
    await hass.services.async_call("recorder", "purge", service_data=service_data)
    await hass.async_block_till_done()

    # Small wait for recorder thread
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        events = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(TEST_EVENT_TYPES))
        )
        statistics = session.query(StatisticsShortTerm)
        recorder_runs = session.query(RecorderRuns)
        statistics_runs = session.query(StatisticsRuns)

        # we should only have 2 states, events and statistics left after purging
        assert states.count() == 2
        assert events.count() == 2
        assert statistics.count() == 2

        # now we should only have 3 recorder runs left
        runs = recorder_runs.all()
        assert_recorder_runs_equal(runs[0], runs_before_purge[0])
        assert_recorder_runs_equal(runs[1], runs_before_purge[5])
        assert_recorder_runs_equal(runs[2], runs_before_purge[6])

        # now we should only have 3 statistics runs left
        runs = statistics_runs.all()
        assert_statistic_runs_equal(runs[0], statistic_runs_before_purge[0])
        assert_statistic_runs_equal(runs[1], statistic_runs_before_purge[5])
        assert_statistic_runs_equal(runs[2], statistic_runs_before_purge[6])

        assert "EVENT_TEST_PURGE" not in (event.event_type for event in events.all())

    # run purge method - correct service data, with repack
    service_data["repack"] = True
    await hass.services.async_call("recorder", "purge", service_data=service_data)
    await hass.async_block_till_done()
    await async_wait_purge_done(hass)
    assert (
        "Vacuuming SQL DB to free space" in caplog.text
        or "Optimizing SQL DB to free space" in caplog.text
    )


@pytest.mark.parametrize("use_sqlite", [True, False], indirect=True)
async def test_purge_edge_case(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    use_sqlite: bool,
) -> None:
    """Test states and events are purged even if they occurred shortly before purge_before."""

    async def _add_db_entries(hass: HomeAssistant, timestamp: datetime) -> None:
        with session_scope(hass=hass) as session:
            session.add(
                Events(
                    event_id=1001,
                    event_type="EVENT_TEST_PURGE",
                    event_data="{}",
                    origin="LOCAL",
                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                )
            )
            session.add(
                States(
                    entity_id="test.recorder2",
                    state="purgeme",
                    attributes="{}",
                    last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                    last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                    event_id=1001,
                    attributes_id=1002,
                )
            )
            session.add(
                StateAttributes(
                    shared_attrs="{}",
                    hash=1234,
                    attributes_id=1002,
                )
            )
            instance = recorder.get_instance(hass)
            convert_pending_events_to_event_types(instance, session)
            convert_pending_states_to_meta(instance, session)

    await async_setup_recorder_instance(hass, None)
    await async_wait_purge_done(hass)

    service_data = {"keep_days": 2}
    timestamp = dt_util.utcnow() - timedelta(days=2, minutes=1)

    await _add_db_entries(hass, timestamp)
    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 1

        state_attributes = session.query(StateAttributes)
        assert state_attributes.count() == 1

        events = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(TEST_EVENT_TYPES))
        )
        assert events.count() == 1

    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await hass.async_block_till_done()

    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 0
        events = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(TEST_EVENT_TYPES))
        )
        assert events.count() == 0


async def test_purge_cutoff_date(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
) -> None:
    """Test states and events are purged only if they occurred before "now() - keep_days"."""

    async def _add_db_entries(hass: HomeAssistant, cutoff: datetime, rows: int) -> None:
        timestamp_keep = cutoff
        timestamp_purge = cutoff - timedelta(microseconds=1)

        with session_scope(hass=hass) as session:
            session.add(
                Events(
                    event_id=1000,
                    event_type="KEEP",
                    event_data="{}",
                    origin="LOCAL",
                    time_fired_ts=dt_util.utc_to_timestamp(timestamp_keep),
                )
            )
            session.add(
                States(
                    entity_id="test.cutoff",
                    state="keep",
                    attributes="{}",
                    last_changed_ts=dt_util.utc_to_timestamp(timestamp_keep),
                    last_updated_ts=dt_util.utc_to_timestamp(timestamp_keep),
                    event_id=1000,
                    attributes_id=1000,
                )
            )
            session.add(
                StateAttributes(
                    shared_attrs="{}",
                    hash=1234,
                    attributes_id=1000,
                )
            )
            for row in range(1, rows):
                session.add(
                    Events(
                        event_id=1000 + row,
                        event_type="PURGE",
                        event_data="{}",
                        origin="LOCAL",
                        time_fired_ts=dt_util.utc_to_timestamp(timestamp_purge),
                    )
                )
                session.add(
                    States(
                        entity_id="test.cutoff",
                        state="purge",
                        attributes="{}",
                        last_changed_ts=dt_util.utc_to_timestamp(timestamp_purge),
                        last_updated_ts=dt_util.utc_to_timestamp(timestamp_purge),
                        event_id=1000 + row,
                        attributes_id=1000 + row,
                    )
                )
                session.add(
                    StateAttributes(
                        shared_attrs="{}",
                        hash=1234,
                        attributes_id=1000 + row,
                    )
                )
            convert_pending_events_to_event_types(instance, session)
            convert_pending_states_to_meta(instance, session)

    instance = await async_setup_recorder_instance(hass, None)
    await async_wait_purge_done(hass)

    service_data = {"keep_days": 2}

    # Force multiple purge batches to be run
    rows = 999
    cutoff = dt_util.utcnow() - timedelta(days=service_data["keep_days"])
    await _add_db_entries(hass, cutoff, rows)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.filter(States.state == "purge").count() == rows - 1
        assert states.filter(States.state == "keep").count() == 1
        assert (
            state_attributes.outerjoin(
                States, StateAttributes.attributes_id == States.attributes_id
            )
            .filter(States.state == "keep")
            .count()
            == 1
        )
        assert (
            session.query(Events)
            .filter(Events.event_type_id.in_(select_event_type_ids(("PURGE",))))
            .count()
            == rows - 1
        )
        assert (
            session.query(Events)
            .filter(Events.event_type_id.in_(select_event_type_ids(("KEEP",))))
            .count()
            == 1
        )

    instance.queue_task(PurgeTask(cutoff, repack=False, apply_filter=False))
    await hass.async_block_till_done()
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        session.query(Events)
        assert states.filter(States.state == "purge").count() == 0
        assert (
            state_attributes.outerjoin(
                States, StateAttributes.attributes_id == States.attributes_id
            )
            .filter(States.state == "purge")
            .count()
            == 0
        )
        assert states.filter(States.state == "keep").count() == 1
        assert (
            state_attributes.outerjoin(
                States, StateAttributes.attributes_id == States.attributes_id
            )
            .filter(States.state == "keep")
            .count()
            == 1
        )
        assert (
            session.query(Events)
            .filter(Events.event_type_id.in_(select_event_type_ids(("PURGE",))))
            .count()
            == 0
        )
        assert (
            session.query(Events)
            .filter(Events.event_type_id.in_(select_event_type_ids(("KEEP",))))
            .count()
            == 1
        )

    # Make sure we can purge everything
    instance.queue_task(PurgeTask(dt_util.utcnow(), repack=False, apply_filter=False))
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.count() == 0
        assert state_attributes.count() == 0

    # Make sure we can purge everything when the db is already empty
    instance.queue_task(PurgeTask(dt_util.utcnow(), repack=False, apply_filter=False))
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.count() == 0
        assert state_attributes.count() == 0


@pytest.mark.parametrize("use_sqlite", [True, False], indirect=True)
async def test_purge_filtered_states(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    use_sqlite: bool,
) -> None:
    """Test filtered states are purged."""
    config: ConfigType = {"exclude": {"entities": ["sensor.excluded"]}}
    instance = await async_setup_recorder_instance(hass, config)
    assert instance.entity_filter("sensor.excluded") is False

    def _add_db_entries(hass: HomeAssistant) -> None:
        with session_scope(hass=hass) as session:
            # Add states and state_changed events that should be purged
            for days in range(1, 4):
                timestamp = dt_util.utcnow() - timedelta(days=days)
                for event_id in range(1000, 1020):
                    _add_state_with_state_attributes(
                        session,
                        "sensor.excluded",
                        "purgeme",
                        timestamp,
                        event_id * days,
                    )
            # Add state **without** state_changed event that should be purged
            timestamp = dt_util.utcnow() - timedelta(days=1)
            session.add(
                States(
                    entity_id="sensor.excluded",
                    state="purgeme",
                    attributes="{}",
                    last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                    last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                )
            )
            # Add states and state_changed events that should be keeped
            timestamp = dt_util.utcnow() - timedelta(days=2)
            for event_id in range(200, 210):
                _add_state_with_state_attributes(
                    session,
                    "sensor.keep",
                    "keep",
                    timestamp,
                    event_id,
                )
            # Add states with linked old_state_ids that need to be handled
            timestamp = dt_util.utcnow() - timedelta(days=0)
            state_attrs = StateAttributes(
                hash=0,
                shared_attrs=json.dumps(
                    {"sensor.linked_old_state_id": "sensor.linked_old_state_id"}
                ),
            )
            state_1 = States(
                entity_id="sensor.linked_old_state_id",
                state="keep",
                attributes="{}",
                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                old_state_id=1,
                state_attributes=state_attrs,
            )
            timestamp = dt_util.utcnow() - timedelta(days=4)
            state_2 = States(
                entity_id="sensor.linked_old_state_id",
                state="keep",
                attributes="{}",
                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                old_state_id=2,
                state_attributes=state_attrs,
            )
            state_3 = States(
                entity_id="sensor.linked_old_state_id",
                state="keep",
                attributes="{}",
                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                old_state_id=62,  # keep
                state_attributes=state_attrs,
            )
            session.add_all((state_attrs, state_1, state_2, state_3))
            # Add event that should be keeped
            session.add(
                Events(
                    event_id=100,
                    event_type="EVENT_KEEP",
                    event_data="{}",
                    origin="LOCAL",
                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                )
            )
            convert_pending_states_to_meta(instance, session)
            convert_pending_events_to_event_types(instance, session)

    service_data = {"keep_days": 10}
    _add_db_entries(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 74
        events_keep = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_KEEP",)))
        )
        assert events_keep.count() == 1

    # Normal purge doesn't remove excluded entities
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await hass.async_block_till_done()

    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 74
        events_keep = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_KEEP",)))
        )
        assert events_keep.count() == 1

    # Test with 'apply_filter' = True
    service_data["apply_filter"] = True
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await hass.async_block_till_done()

    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 13
        events_keep = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_KEEP",)))
        )
        assert events_keep.count() == 1

        states_sensor_excluded = (
            session.query(States)
            .outerjoin(StatesMeta, States.metadata_id == StatesMeta.metadata_id)
            .filter(StatesMeta.entity_id == "sensor.excluded")
        )
        assert states_sensor_excluded.count() == 0

        assert (
            session.query(States).filter(States.state_id == 72).first().old_state_id
            is None
        )
        assert (
            session.query(States).filter(States.state_id == 72).first().attributes_id
            == 71
        )
        assert (
            session.query(States).filter(States.state_id == 73).first().old_state_id
            is None
        )
        assert (
            session.query(States).filter(States.state_id == 73).first().attributes_id
            == 71
        )

        final_keep_state = session.query(States).filter(States.state_id == 74).first()
        assert final_keep_state.old_state_id == 62  # should have been kept
        assert final_keep_state.attributes_id == 71

        assert session.query(StateAttributes).count() == 11

    # Do it again to make sure nothing changes
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        final_keep_state = session.query(States).filter(States.state_id == 74).first()
        assert final_keep_state.old_state_id == 62  # should have been kept
        assert final_keep_state.attributes_id == 71

        assert session.query(StateAttributes).count() == 11

    service_data = {"keep_days": 0}
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        remaining = list(session.query(States))
        for state in remaining:
            assert state.event_id is None
        assert len(remaining) == 0
        assert session.query(StateAttributes).count() == 0


@pytest.mark.parametrize("use_sqlite", [True, False], indirect=True)
async def test_purge_filtered_states_to_empty(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    use_sqlite: bool,
) -> None:
    """Test filtered states are purged all the way to an empty db."""
    config: ConfigType = {"exclude": {"entities": ["sensor.excluded"]}}
    instance = await async_setup_recorder_instance(hass, config)
    assert instance.entity_filter("sensor.excluded") is False

    def _add_db_entries(hass: HomeAssistant) -> None:
        with session_scope(hass=hass) as session:
            # Add states and state_changed events that should be purged
            for days in range(1, 4):
                timestamp = dt_util.utcnow() - timedelta(days=days)
                for event_id in range(1000, 1020):
                    _add_state_with_state_attributes(
                        session,
                        "sensor.excluded",
                        "purgeme",
                        timestamp,
                        event_id * days,
                    )
            convert_pending_states_to_meta(instance, session)

    service_data = {"keep_days": 10}
    _add_db_entries(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.count() == 60
        assert state_attributes.count() == 60

    # Test with 'apply_filter' = True
    service_data["apply_filter"] = True
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.count() == 0
        assert state_attributes.count() == 0

    # Do it again to make sure nothing changes
    # Why do we do this? Should we check the end result?
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)


@pytest.mark.parametrize("use_sqlite", [True, False], indirect=True)
async def test_purge_without_state_attributes_filtered_states_to_empty(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
    use_sqlite: bool,
) -> None:
    """Test filtered legacy states without state attributes are purged all the way to an empty db."""
    config: ConfigType = {"exclude": {"entities": ["sensor.old_format"]}}
    instance = await async_setup_recorder_instance(hass, config)
    assert instance.entity_filter("sensor.old_format") is False

    def _add_db_entries(hass: HomeAssistant) -> None:
        with session_scope(hass=hass) as session:
            # Add states and state_changed events that should be purged
            # in the legacy format
            timestamp = dt_util.utcnow() - timedelta(days=5)
            event_id = 1021
            session.add(
                States(
                    entity_id="sensor.old_format",
                    state=STATE_ON,
                    attributes=json.dumps({"old": "not_using_state_attributes"}),
                    last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                    last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                    event_id=event_id,
                    state_attributes=None,
                )
            )
            session.add(
                Events(
                    event_id=event_id,
                    event_type=EVENT_STATE_CHANGED,
                    event_data="{}",
                    origin="LOCAL",
                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                )
            )
            session.add(
                Events(
                    event_id=event_id + 1,
                    event_type=EVENT_THEMES_UPDATED,
                    event_data="{}",
                    origin="LOCAL",
                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                )
            )
            convert_pending_states_to_meta(instance, session)
            convert_pending_events_to_event_types(instance, session)

    service_data = {"keep_days": 10}
    _add_db_entries(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.count() == 1
        assert state_attributes.count() == 0

    # Test with 'apply_filter' = True
    service_data["apply_filter"] = True
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        states = session.query(States)
        state_attributes = session.query(StateAttributes)
        assert states.count() == 0
        assert state_attributes.count() == 0

    # Do it again to make sure nothing changes
    # Why do we do this? Should we check the end result?
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)


async def test_purge_filtered_events(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
) -> None:
    """Test filtered events are purged."""
    config: ConfigType = {"exclude": {"event_types": ["EVENT_PURGE"]}}
    instance = await async_setup_recorder_instance(hass, config)
    await async_wait_recording_done(hass)

    def _add_db_entries(hass: HomeAssistant) -> None:
        with session_scope(hass=hass) as session:
            # Add events that should be purged
            for days in range(1, 4):
                timestamp = dt_util.utcnow() - timedelta(days=days)
                for event_id in range(1000, 1020):
                    session.add(
                        Events(
                            event_id=event_id * days,
                            event_type="EVENT_PURGE",
                            event_data="{}",
                            origin="LOCAL",
                            time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                        )
                    )

            # Add states and state_changed events that should be keeped
            timestamp = dt_util.utcnow() - timedelta(days=1)
            for event_id in range(200, 210):
                _add_state_with_state_attributes(
                    session,
                    "sensor.keep",
                    "keep",
                    timestamp,
                    event_id,
                )
            convert_pending_events_to_event_types(instance, session)
            convert_pending_states_to_meta(instance, session)

    service_data = {"keep_days": 10}
    await instance.async_add_executor_job(_add_db_entries, hass)
    await async_wait_recording_done(hass)

    with session_scope(hass=hass, read_only=True) as session:
        events_purge = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_PURGE",)))
        )
        states = session.query(States)
        assert events_purge.count() == 60
        assert states.count() == 10

    # Normal purge doesn't remove excluded events
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await hass.async_block_till_done()

    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass, read_only=True) as session:
        events_purge = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_PURGE",)))
        )
        states = session.query(States)
        assert events_purge.count() == 60
        assert states.count() == 10

    # Test with 'apply_filter' = True
    service_data["apply_filter"] = True
    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await hass.async_block_till_done()

    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    with session_scope(hass=hass, read_only=True) as session:
        events_purge = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_PURGE",)))
        )
        states = session.query(States)
        assert events_purge.count() == 0
        assert states.count() == 10


async def test_purge_filtered_events_state_changed(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
) -> None:
    """Test filtered state_changed events are purged. This should also remove all states."""
    config: ConfigType = {
        "exclude": {
            "event_types": ["excluded_event"],
            "entities": ["sensor.excluded", "sensor.old_format"],
        }
    }
    instance = await async_setup_recorder_instance(hass, config)
    # Assert entity_id is NOT excluded
    assert instance.entity_filter("sensor.excluded") is False
    assert instance.entity_filter("sensor.old_format") is False
    assert instance.entity_filter("sensor.keep") is True
    assert "excluded_event" in instance.exclude_event_types

    def _add_db_entries(hass: HomeAssistant) -> None:
        with session_scope(hass=hass) as session:
            # Add states and state_changed events that should be purged
            for days in range(1, 4):
                timestamp = dt_util.utcnow() - timedelta(days=days)
                for event_id in range(1000, 1020):
                    _add_state_with_state_attributes(
                        session,
                        "sensor.excluded",
                        "purgeme",
                        timestamp,
                        event_id * days,
                    )
            # Add events that should be keeped
            timestamp = dt_util.utcnow() - timedelta(days=1)
            for event_id in range(200, 210):
                session.add(
                    Events(
                        event_id=event_id,
                        event_type="EVENT_KEEP",
                        event_data="{}",
                        origin="LOCAL",
                        time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                    )
                )
            # Add states with linked old_state_ids that need to be handled
            timestamp = dt_util.utcnow() - timedelta(days=0)
            state_1 = States(
                entity_id="sensor.linked_old_state_id",
                state="keep",
                attributes="{}",
                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                old_state_id=1,
            )
            timestamp = dt_util.utcnow() - timedelta(days=4)
            state_2 = States(
                entity_id="sensor.linked_old_state_id",
                state="keep",
                attributes="{}",
                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                old_state_id=2,
            )
            state_3 = States(
                entity_id="sensor.linked_old_state_id",
                state="keep",
                attributes="{}",
                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                old_state_id=62,  # keep
            )
            session.add_all((state_1, state_2, state_3))
            session.add(
                Events(
                    event_id=231,
                    event_type="excluded_event",
                    event_data="{}",
                    origin="LOCAL",
                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                )
            )
            session.add(
                States(
                    entity_id="sensor.old_format",
                    state="remove",
                    attributes="{}",
                    last_changed_ts=dt_util.utc_to_timestamp(timestamp),
                    last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                )
            )
            convert_pending_events_to_event_types(instance, session)
            convert_pending_states_to_meta(instance, session)

    service_data = {"keep_days": 10, "apply_filter": True}
    _add_db_entries(hass)

    with session_scope(hass=hass) as session:
        events_keep = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_KEEP",)))
        )
        events_purge = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("excluded_event",)))
        )
        states = session.query(States)

        assert events_keep.count() == 10
        assert events_purge.count() == 1
        assert states.count() == 64

    await hass.services.async_call(recorder.DOMAIN, SERVICE_PURGE, service_data)
    await hass.async_block_till_done()

    for _ in range(4):
        await async_recorder_block_till_done(hass)
        await async_wait_purge_done(hass)

    with session_scope(hass=hass) as session:
        events_keep = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("EVENT_KEEP",)))
        )
        events_purge = session.query(Events).filter(
            Events.event_type_id.in_(select_event_type_ids(("excluded_event",)))
        )
        states = session.query(States)

        assert events_keep.count() == 10
        assert events_purge.count() == 0
        assert states.count() == 3

        assert (
            session.query(States).filter(States.state_id == 61).first().old_state_id
            is None
        )
        assert (
            session.query(States).filter(States.state_id == 62).first().old_state_id
            is None
        )
        assert (
            session.query(States).filter(States.state_id == 63).first().old_state_id
            == 62
        )  # should have been kept


async def test_purge_entities(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test purging of specific entities."""
    instance = await async_setup_recorder_instance(hass)

    async def _purge_entities(hass, entity_ids, domains, entity_globs):
        service_data = {
            "entity_id": entity_ids,
            "domains": domains,
            "entity_globs": entity_globs,
        }

        await hass.services.async_call(
            recorder.DOMAIN, SERVICE_PURGE_ENTITIES, service_data
        )
        await hass.async_block_till_done()

        await async_recorder_block_till_done(hass)
        await async_wait_purge_done(hass)

    def _add_purge_records(hass: HomeAssistant) -> None:
        with session_scope(hass=hass) as session:
            # Add states and state_changed events that should be purged
            for days in range(1, 4):
                timestamp = dt_util.utcnow() - timedelta(days=days)
                for event_id in range(1000, 1020):
                    _add_state_with_state_attributes(
                        session,
                        "sensor.purge_entity",
                        "purgeme",
                        timestamp,
                        event_id * days,
                    )
                timestamp = dt_util.utcnow() - timedelta(days=days)
                for event_id in range(10000, 10020):
                    _add_state_with_state_attributes(
                        session,
                        "purge_domain.entity",
                        "purgeme",
                        timestamp,
                        event_id * days,
                    )
                timestamp = dt_util.utcnow() - timedelta(days=days)
                for event_id in range(100000, 100020):
                    _add_state_with_state_attributes(
                        session,
                        "binary_sensor.purge_glob",
                        "purgeme",
                        timestamp,
                        event_id * days,
                    )
            convert_pending_states_to_meta(instance, session)
            convert_pending_events_to_event_types(instance, session)

    def _add_keep_records(hass: HomeAssistant) -> None:
        with session_scope(hass=hass) as session:
            # Add states and state_changed events that should be kept
            timestamp = dt_util.utcnow() - timedelta(days=2)
            for event_id in range(200, 210):
                _add_state_with_state_attributes(
                    session,
                    "sensor.keep",
                    "keep",
                    timestamp,
                    event_id,
                )
            convert_pending_states_to_meta(instance, session)
            convert_pending_events_to_event_types(instance, session)

    _add_purge_records(hass)
    _add_keep_records(hass)

    # Confirm standard service call
    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 190

    await _purge_entities(hass, "sensor.purge_entity", "purge_domain", "*purge_glob")

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 10

        states_sensor_kept = (
            session.query(States)
            .outerjoin(StatesMeta, States.metadata_id == StatesMeta.metadata_id)
            .filter(StatesMeta.entity_id == "sensor.keep")
        )
        assert states_sensor_kept.count() == 10

    _add_purge_records(hass)

    # Confirm each parameter purges only the associated records
    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 190

    await _purge_entities(hass, "sensor.purge_entity", [], [])

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 130

    await _purge_entities(hass, [], "purge_domain", [])

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 70

    await _purge_entities(hass, [], [], "*purge_glob")

    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 10

        states_sensor_kept = (
            session.query(States)
            .outerjoin(StatesMeta, States.metadata_id == StatesMeta.metadata_id)
            .filter(StatesMeta.entity_id == "sensor.keep")
        )
        assert states_sensor_kept.count() == 10

        # sensor.keep should remain in the StatesMeta table
        states_meta_remain = session.query(StatesMeta).filter(
            StatesMeta.entity_id == "sensor.keep"
        )
        assert states_meta_remain.count() == 1

        # sensor.purge_entity should be removed from the StatesMeta table
        states_meta_remain = session.query(StatesMeta).filter(
            StatesMeta.entity_id == "sensor.purge_entity"
        )
        assert states_meta_remain.count() == 0

    _add_purge_records(hass)

    # Confirm calling service without arguments matches all records (default filter behavior)
    with session_scope(hass=hass) as session:
        states = session.query(States)
        assert states.count() == 190

    await _purge_entities(hass, [], [], [])

    with session_scope(hass=hass, read_only=True) as session:
        states = session.query(States)
        assert states.count() == 0

        # The states_meta table should be empty
        states_meta_remain = session.query(StatesMeta)
        assert states_meta_remain.count() == 0


async def _add_test_states(hass: HomeAssistant, wait_recording_done: bool = True):
    """Add multiple states to the db for testing."""
    utcnow = dt_util.utcnow()
    five_days_ago = utcnow - timedelta(days=5)
    eleven_days_ago = utcnow - timedelta(days=11)
    base_attributes = {"test_attr": 5, "test_attr_10": "nice"}

    async def set_state(entity_id, state, **kwargs):
        """Set the state."""
        hass.states.async_set(entity_id, state, **kwargs)
        if wait_recording_done:
            await hass.async_block_till_done()
            await async_wait_recording_done(hass)

    with freeze_time() as freezer:
        for event_id in range(6):
            if event_id < 2:
                timestamp = eleven_days_ago
                state = f"autopurgeme_{event_id}"
                attributes = {"autopurgeme": True, **base_attributes}
            elif event_id < 4:
                timestamp = five_days_ago
                state = f"purgeme_{event_id}"
                attributes = {"purgeme": True, **base_attributes}
            else:
                timestamp = utcnow
                state = f"dontpurgeme_{event_id}"
                attributes = {"dontpurgeme": True, **base_attributes}

            freezer.move_to(timestamp)
            await set_state("test.recorder2", state, attributes=attributes)


async def _add_test_events(hass: HomeAssistant, iterations: int = 1):
    """Add a few events for testing."""
    utcnow = dt_util.utcnow()
    five_days_ago = utcnow - timedelta(days=5)
    eleven_days_ago = utcnow - timedelta(days=11)
    event_data = {"test_attr": 5, "test_attr_10": "nice"}
    # Make sure recording is done before freezing time
    # because the time freeze can affect the recorder
    # thread as well can cause the test to fail
    await async_wait_recording_done(hass)

    with freeze_time() as freezer:
        for _ in range(iterations):
            for event_id in range(6):
                if event_id < 2:
                    timestamp = eleven_days_ago
                    event_type = "EVENT_TEST_AUTOPURGE"
                elif event_id < 4:
                    timestamp = five_days_ago
                    event_type = "EVENT_TEST_PURGE"
                else:
                    timestamp = utcnow
                    event_type = "EVENT_TEST"
                freezer.move_to(timestamp)
                hass.bus.async_fire(event_type, event_data)

    await async_wait_recording_done(hass)


async def _add_test_statistics(hass: HomeAssistant):
    """Add multiple statistics to the db for testing."""
    utcnow = dt_util.utcnow()
    five_days_ago = utcnow - timedelta(days=5)
    eleven_days_ago = utcnow - timedelta(days=11)

    await hass.async_block_till_done()
    await async_wait_recording_done(hass)

    with session_scope(hass=hass) as session:
        for event_id in range(6):
            if event_id < 2:
                timestamp = eleven_days_ago
                state = "-11"
            elif event_id < 4:
                timestamp = five_days_ago
                state = "-5"
            else:
                timestamp = utcnow
                state = "0"

            session.add(
                StatisticsShortTerm(
                    start_ts=timestamp.timestamp(),
                    state=state,
                )
            )


async def _add_test_recorder_runs(hass: HomeAssistant):
    """Add a few recorder_runs for testing."""
    utcnow = dt_util.utcnow()
    five_days_ago = utcnow - timedelta(days=5)
    eleven_days_ago = utcnow - timedelta(days=11)

    await hass.async_block_till_done()
    await async_wait_recording_done(hass)

    with session_scope(hass=hass) as session:
        for rec_id in range(6):
            if rec_id < 2:
                timestamp = eleven_days_ago
            elif rec_id < 4:
                timestamp = five_days_ago
            else:
                timestamp = utcnow

            session.add(
                RecorderRuns(
                    start=timestamp,
                    created=dt_util.utcnow(),
                    end=timestamp + timedelta(days=1),
                )
            )


async def _add_test_statistics_runs(hass: HomeAssistant):
    """Add a few recorder_runs for testing."""
    utcnow = dt_util.utcnow()
    five_days_ago = utcnow - timedelta(days=5)
    eleven_days_ago = utcnow - timedelta(days=11)

    await hass.async_block_till_done()
    await async_wait_recording_done(hass)

    with session_scope(hass=hass) as session:
        for rec_id in range(6):
            if rec_id < 2:
                timestamp = eleven_days_ago
            elif rec_id < 4:
                timestamp = five_days_ago
            else:
                timestamp = utcnow

            session.add(
                StatisticsRuns(
                    start=timestamp,
                )
            )


def _add_state_without_event_linkage(
    session: Session,
    entity_id: str,
    state: str,
    timestamp: datetime,
):
    state_attrs = StateAttributes(
        hash=1234, shared_attrs=json.dumps({entity_id: entity_id})
    )
    session.add(state_attrs)
    session.add(
        States(
            entity_id=entity_id,
            state=state,
            attributes=None,
            last_changed_ts=dt_util.utc_to_timestamp(timestamp),
            last_updated_ts=dt_util.utc_to_timestamp(timestamp),
            event_id=None,
            state_attributes=state_attrs,
        )
    )


def _add_state_with_state_attributes(
    session: Session,
    entity_id: str,
    state: str,
    timestamp: datetime,
    event_id: int,
) -> None:
    """Add state and state_changed event to database for testing."""
    state_attrs = StateAttributes(
        hash=event_id, shared_attrs=json.dumps({entity_id: entity_id})
    )
    session.add(state_attrs)
    session.add(
        States(
            entity_id=entity_id,
            state=state,
            attributes=None,
            last_changed_ts=dt_util.utc_to_timestamp(timestamp),
            last_updated_ts=dt_util.utc_to_timestamp(timestamp),
            event_id=event_id,
            state_attributes=state_attrs,
        )
    )


@pytest.mark.timeout(30)
async def test_purge_many_old_events(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting old events."""
    old_events_count = 5
    instance = await async_setup_recorder_instance(hass)

    with (
        patch.object(instance, "max_bind_vars", old_events_count),
        patch.object(instance.database_engine, "max_bind_vars", old_events_count),
    ):
        await _add_test_events(hass, old_events_count)

        with session_scope(hass=hass) as session:
            events = session.query(Events).filter(
                Events.event_type_id.in_(select_event_type_ids(TEST_EVENT_TYPES))
            )
            assert events.count() == old_events_count * 6

            purge_before = dt_util.utcnow() - timedelta(days=4)

            # run purge_old_data()
            finished = purge_old_data(
                instance,
                purge_before,
                repack=False,
                states_batch_size=3,
                events_batch_size=3,
            )
            assert not finished
            assert events.count() == old_events_count * 3

            # we should only have 2 groups of events left
            finished = purge_old_data(
                instance,
                purge_before,
                repack=False,
                states_batch_size=3,
                events_batch_size=3,
            )
            assert finished
            assert events.count() == old_events_count * 2

            # we should now purge everything
            finished = purge_old_data(
                instance,
                dt_util.utcnow(),
                repack=False,
                states_batch_size=20,
                events_batch_size=20,
            )
            assert finished
            assert events.count() == 0


async def test_purge_old_events_purges_the_event_type_ids(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting old events purges event type ids."""
    instance = await async_setup_recorder_instance(hass)
    assert instance.event_type_manager.active is True

    utcnow = dt_util.utcnow()
    five_days_ago = utcnow - timedelta(days=5)
    eleven_days_ago = utcnow - timedelta(days=11)
    far_past = utcnow - timedelta(days=1000)

    await hass.async_block_till_done()
    await async_wait_recording_done(hass)

    def _insert_events():
        with session_scope(hass=hass) as session:
            event_type_test_auto_purge = EventTypes(event_type="EVENT_TEST_AUTOPURGE")
            event_type_test_purge = EventTypes(event_type="EVENT_TEST_PURGE")
            event_type_test = EventTypes(event_type="EVENT_TEST")
            event_type_unused = EventTypes(event_type="EVENT_TEST_UNUSED")
            session.add_all(
                (
                    event_type_test_auto_purge,
                    event_type_test_purge,
                    event_type_test,
                    event_type_unused,
                )
            )
            session.flush()
            for _ in range(5):
                for event_id in range(6):
                    if event_id < 2:
                        timestamp = eleven_days_ago
                        event_type = event_type_test_auto_purge
                    elif event_id < 4:
                        timestamp = five_days_ago
                        event_type = event_type_test_purge
                    else:
                        timestamp = utcnow
                        event_type = event_type_test

                    session.add(
                        Events(
                            event_type=None,
                            event_type_id=event_type.event_type_id,
                            time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                        )
                    )
            return instance.event_type_manager.get_many(
                [
                    "EVENT_TEST_AUTOPURGE",
                    "EVENT_TEST_PURGE",
                    "EVENT_TEST",
                    "EVENT_TEST_UNUSED",
                ],
                session,
            )

    event_type_to_id = await instance.async_add_executor_job(_insert_events)
    test_event_type_ids = event_type_to_id.values()
    with session_scope(hass=hass) as session:
        events = session.query(Events).where(
            Events.event_type_id.in_(test_event_type_ids)
        )
        event_types = session.query(EventTypes).where(
            EventTypes.event_type_id.in_(test_event_type_ids)
        )

        assert events.count() == 30
        assert event_types.count() == 4

        # run purge_old_data()
        finished = purge_old_data(
            instance,
            far_past,
            repack=False,
        )
        assert finished
        assert events.count() == 30
        # We should remove the unused event type
        assert event_types.count() == 3

        assert "EVENT_TEST_UNUSED" not in instance.event_type_manager._id_map

        # we should only have 10 events left since
        # only one event type was recorded now
        finished = purge_old_data(
            instance,
            utcnow,
            repack=False,
        )
        assert finished
        assert events.count() == 10
        assert event_types.count() == 1

        # Purge everything
        finished = purge_old_data(
            instance,
            utcnow + timedelta(seconds=1),
            repack=False,
        )
        assert finished
        assert events.count() == 0
        assert event_types.count() == 0


async def test_purge_old_states_purges_the_state_metadata_ids(
    async_setup_recorder_instance: RecorderInstanceGenerator, hass: HomeAssistant
) -> None:
    """Test deleting old states purges state metadata_ids."""
    instance = await async_setup_recorder_instance(hass)
    assert instance.states_meta_manager.active is True

    utcnow = dt_util.utcnow()
    five_days_ago = utcnow - timedelta(days=5)
    eleven_days_ago = utcnow - timedelta(days=11)
    far_past = utcnow - timedelta(days=1000)

    await hass.async_block_till_done()
    await async_wait_recording_done(hass)

    def _insert_states():
        with session_scope(hass=hass) as session:
            states_meta_sensor_one = StatesMeta(entity_id="sensor.one")
            states_meta_sensor_two = StatesMeta(entity_id="sensor.two")
            states_meta_sensor_three = StatesMeta(entity_id="sensor.three")
            states_meta_sensor_unused = StatesMeta(entity_id="sensor.unused")
            session.add_all(
                (
                    states_meta_sensor_one,
                    states_meta_sensor_two,
                    states_meta_sensor_three,
                    states_meta_sensor_unused,
                )
            )
            session.flush()
            for _ in range(5):
                for event_id in range(6):
                    if event_id < 2:
                        timestamp = eleven_days_ago
                        metadata_id = states_meta_sensor_one.metadata_id
                    elif event_id < 4:
                        timestamp = five_days_ago
                        metadata_id = states_meta_sensor_two.metadata_id
                    else:
                        timestamp = utcnow
                        metadata_id = states_meta_sensor_three.metadata_id

                    session.add(
                        States(
                            metadata_id=metadata_id,
                            state="any",
                            last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                        )
                    )
            return instance.states_meta_manager.get_many(
                ["sensor.one", "sensor.two", "sensor.three", "sensor.unused"],
                session,
                True,
            )

    entity_id_to_metadata_id = await instance.async_add_executor_job(_insert_states)
    test_metadata_ids = entity_id_to_metadata_id.values()
    with session_scope(hass=hass) as session:
        states = session.query(States).where(States.metadata_id.in_(test_metadata_ids))
        states_meta = session.query(StatesMeta).where(
            StatesMeta.metadata_id.in_(test_metadata_ids)
        )

        assert states.count() == 30
        assert states_meta.count() == 4

        # run purge_old_data()
        finished = purge_old_data(
            instance,
            far_past,
            repack=False,
        )
        assert finished
        assert states.count() == 30
        # We should remove the unused entity_id
        assert states_meta.count() == 3

        assert "sensor.unused" not in instance.event_type_manager._id_map

        # we should only have 10 states left since
        # only one event type was recorded now
        finished = purge_old_data(
            instance,
            utcnow,
            repack=False,
        )
        assert finished
        assert states.count() == 10
        assert states_meta.count() == 1

        # Purge everything
        finished = purge_old_data(
            instance,
            utcnow + timedelta(seconds=1),
            repack=False,
        )
        assert finished
        assert states.count() == 0
        assert states_meta.count() == 0


async def test_purge_entities_keep_days(
    async_setup_recorder_instance: RecorderInstanceGenerator,
    hass: HomeAssistant,
) -> None:
    """Test purging states with an entity filter and keep_days."""
    instance = await async_setup_recorder_instance(hass, {})
    await hass.async_block_till_done()
    await async_wait_recording_done(hass)
    start = dt_util.utcnow()
    two_days_ago = start - timedelta(days=2)
    one_week_ago = start - timedelta(days=7)
    one_month_ago = start - timedelta(days=30)
    with freeze_time(one_week_ago):
        hass.states.async_set("sensor.keep", "initial")
        hass.states.async_set("sensor.purge", "initial")

    await async_wait_recording_done(hass)

    with freeze_time(two_days_ago):
        hass.states.async_set("sensor.purge", "two_days_ago")

    await async_wait_recording_done(hass)

    hass.states.async_set("sensor.purge", "now")
    hass.states.async_set("sensor.keep", "now")
    await async_recorder_block_till_done(hass)

    states = await instance.async_add_executor_job(
        get_significant_states,
        hass,
        one_month_ago,
        None,
        ["sensor.keep", "sensor.purge"],
    )
    assert len(states["sensor.keep"]) == 2
    assert len(states["sensor.purge"]) == 3

    await hass.services.async_call(
        recorder.DOMAIN,
        SERVICE_PURGE_ENTITIES,
        {
            "entity_id": "sensor.purge",
            "keep_days": 1,
        },
    )
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    states = await instance.async_add_executor_job(
        get_significant_states,
        hass,
        one_month_ago,
        None,
        ["sensor.keep", "sensor.purge"],
    )
    assert len(states["sensor.keep"]) == 2
    assert len(states["sensor.purge"]) == 1

    await hass.services.async_call(
        recorder.DOMAIN,
        SERVICE_PURGE_ENTITIES,
        {
            "entity_id": "sensor.purge",
        },
    )
    await async_recorder_block_till_done(hass)
    await async_wait_purge_done(hass)

    states = await instance.async_add_executor_job(
        get_significant_states,
        hass,
        one_month_ago,
        None,
        ["sensor.keep", "sensor.purge"],
    )
    assert len(states["sensor.keep"]) == 2
    assert "sensor.purge" not in states
