1
mirror of https://github.com/home-assistant/core synced 2024-09-06 10:29:55 +02:00

Adjust entity filters to make includes stronger than excludes (#74080)

* Adjust entity filters to make includes stronger than excludes

Fixes #59080

* adjust test for stronger entity glob includes

* sync with docs
This commit is contained in:
J. Nick Koston 2022-06-28 11:42:51 -05:00 committed by GitHub
parent 040ece76ab
commit a8349a4866
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 312 additions and 87 deletions

View File

@ -139,49 +139,52 @@ class Filters:
have_exclude = self._have_exclude
have_include = self._have_include
# Case 1 - no includes or excludes - pass all entities
# Case 1 - No filter
# - All entities included
if not have_include and not have_exclude:
return None
# Case 2 - includes, no excludes - only include specified entities
# Case 2 - Only includes
# - Entity listed in entities include: include
# - Otherwise, entity matches domain include: include
# - Otherwise, entity matches glob include: include
# - Otherwise: exclude
if have_include and not have_exclude:
return or_(*includes).self_group()
# Case 3 - excludes, no includes - only exclude specified entities
# Case 3 - Only excludes
# - Entity listed in exclude: exclude
# - Otherwise, entity matches domain exclude: exclude
# - Otherwise, entity matches glob exclude: exclude
# - Otherwise: include
if not have_include and have_exclude:
return not_(or_(*excludes).self_group())
# Case 4 - both includes and excludes specified
# Case 4a - include domain or glob specified
# - if domain is included, pass if entity not excluded
# - if glob is included, pass if entity and domain not excluded
# - if domain and glob are not included, pass if entity is included
# note: if both include domain matches then exclude domains ignored.
# If glob matches then exclude domains and glob checked
# Case 4 - Domain and/or glob includes (may also have excludes)
# - Entity listed in entities include: include
# - Otherwise, entity listed in entities exclude: exclude
# - Otherwise, entity matches glob include: include
# - Otherwise, entity matches glob exclude: exclude
# - Otherwise, entity matches domain include: include
# - Otherwise: exclude
if self.included_domains or self.included_entity_globs:
return or_(
(i_domains & ~(e_entities | e_entity_globs)),
(
~i_domains
& or_(
(i_entity_globs & ~(or_(*excludes))),
(~i_entity_globs & i_entities),
)
),
i_entities,
(~e_entities & (i_entity_globs | (~e_entity_globs & i_domains))),
).self_group()
# Case 4b - exclude domain or glob specified, include has no domain or glob
# In this one case the traditional include logic is inverted. Even though an
# include is specified since its only a list of entity IDs its used only to
# expose specific entities excluded by domain or glob. Any entities not
# excluded are then presumed included. Logic is as follows
# - if domain or glob is excluded, pass if entity is included
# - if domain is not excluded, pass if entity not excluded by ID
# Case 5 - Domain and/or glob excludes (no domain and/or glob includes)
# - Entity listed in entities include: include
# - Otherwise, entity listed in exclude: exclude
# - Otherwise, entity matches glob exclude: exclude
# - Otherwise, entity matches domain exclude: exclude
# - Otherwise: include
if self.excluded_domains or self.excluded_entity_globs:
return (not_(or_(*excludes)) | i_entities).self_group()
# Case 4c - neither include or exclude domain specified
# - Only pass if entity is included. Ignore entity excludes.
# Case 6 - No Domain and/or glob includes or excludes
# - Entity listed in entities include: include
# - Otherwise: exclude
return i_entities
def states_entity_filter(self) -> ClauseList:

View File

@ -145,11 +145,7 @@ def _glob_to_re(glob: str) -> re.Pattern[str]:
def _test_against_patterns(patterns: list[re.Pattern[str]], entity_id: str) -> bool:
"""Test entity against list of patterns, true if any match."""
for pattern in patterns:
if pattern.match(entity_id):
return True
return False
return any(pattern.match(entity_id) for pattern in patterns)
def _convert_globs_to_pattern_list(globs: list[str] | None) -> list[re.Pattern[str]]:
@ -193,7 +189,7 @@ def _generate_filter_from_sets_and_pattern_lists(
return (
entity_id in include_e
or domain in include_d
or bool(include_eg and _test_against_patterns(include_eg, entity_id))
or _test_against_patterns(include_eg, entity_id)
)
def entity_excluded(domain: str, entity_id: str) -> bool:
@ -201,14 +197,19 @@ def _generate_filter_from_sets_and_pattern_lists(
return (
entity_id in exclude_e
or domain in exclude_d
or bool(exclude_eg and _test_against_patterns(exclude_eg, entity_id))
or _test_against_patterns(exclude_eg, entity_id)
)
# Case 1 - no includes or excludes - pass all entities
# Case 1 - No filter
# - All entities included
if not have_include and not have_exclude:
return lambda entity_id: True
# Case 2 - includes, no excludes - only include specified entities
# Case 2 - Only includes
# - Entity listed in entities include: include
# - Otherwise, entity matches domain include: include
# - Otherwise, entity matches glob include: include
# - Otherwise: exclude
if have_include and not have_exclude:
def entity_filter_2(entity_id: str) -> bool:
@ -218,7 +219,11 @@ def _generate_filter_from_sets_and_pattern_lists(
return entity_filter_2
# Case 3 - excludes, no includes - only exclude specified entities
# Case 3 - Only excludes
# - Entity listed in exclude: exclude
# - Otherwise, entity matches domain exclude: exclude
# - Otherwise, entity matches glob exclude: exclude
# - Otherwise: include
if not have_include and have_exclude:
def entity_filter_3(entity_id: str) -> bool:
@ -228,38 +233,36 @@ def _generate_filter_from_sets_and_pattern_lists(
return entity_filter_3
# Case 4 - both includes and excludes specified
# Case 4a - include domain or glob specified
# - if domain is included, pass if entity not excluded
# - if glob is included, pass if entity and domain not excluded
# - if domain and glob are not included, pass if entity is included
# note: if both include domain matches then exclude domains ignored.
# If glob matches then exclude domains and glob checked
# Case 4 - Domain and/or glob includes (may also have excludes)
# - Entity listed in entities include: include
# - Otherwise, entity listed in entities exclude: exclude
# - Otherwise, entity matches glob include: include
# - Otherwise, entity matches glob exclude: exclude
# - Otherwise, entity matches domain include: include
# - Otherwise: exclude
if include_d or include_eg:
def entity_filter_4a(entity_id: str) -> bool:
"""Return filter function for case 4a."""
domain = split_entity_id(entity_id)[0]
if domain in include_d:
return not (
entity_id in exclude_e
or bool(
exclude_eg and _test_against_patterns(exclude_eg, entity_id)
return entity_id in include_e or (
entity_id not in exclude_e
and (
_test_against_patterns(include_eg, entity_id)
or (
split_entity_id(entity_id)[0] in include_d
and not _test_against_patterns(exclude_eg, entity_id)
)
)
if _test_against_patterns(include_eg, entity_id):
return not entity_excluded(domain, entity_id)
return entity_id in include_e
)
return entity_filter_4a
# Case 4b - exclude domain or glob specified, include has no domain or glob
# In this one case the traditional include logic is inverted. Even though an
# include is specified since its only a list of entity IDs its used only to
# expose specific entities excluded by domain or glob. Any entities not
# excluded are then presumed included. Logic is as follows
# - if domain or glob is excluded, pass if entity is included
# - if domain is not excluded, pass if entity not excluded by ID
# Case 5 - Domain and/or glob excludes (no domain and/or glob includes)
# - Entity listed in entities include: include
# - Otherwise, entity listed in exclude: exclude
# - Otherwise, entity matches glob exclude: exclude
# - Otherwise, entity matches domain exclude: exclude
# - Otherwise: include
if exclude_d or exclude_eg:
def entity_filter_4b(entity_id: str) -> bool:
@ -273,6 +276,7 @@ def _generate_filter_from_sets_and_pattern_lists(
return entity_filter_4b
# Case 4c - neither include or exclude domain specified
# - Only pass if entity is included. Ignore entity excludes.
# Case 6 - No Domain and/or glob includes or excludes
# - Entity listed in entities include: include
# - Otherwise: exclude
return lambda entity_id: entity_id in include_e

View File

@ -169,7 +169,7 @@ async def test_filtered_allowlist(hass, mock_client):
FilterTest("light.excluded_test", False),
FilterTest("light.excluded", False),
FilterTest("sensor.included_test", True),
FilterTest("climate.included_test", False),
FilterTest("climate.included_test", True),
]
await _run_filter_tests(hass, tests, mock_client)

View File

@ -176,7 +176,7 @@ async def test_full_batch(hass, entry_with_one_event, mock_create_batch):
FilterTest("light.excluded_test", 0),
FilterTest("light.excluded", 0),
FilterTest("sensor.included_test", 1),
FilterTest("climate.included_test", 0),
FilterTest("climate.included_test", 1),
],
),
(

View File

@ -222,7 +222,7 @@ async def test_filtered_allowlist(hass, mock_client):
FilterTest("light.excluded_test", False),
FilterTest("light.excluded", False),
FilterTest("sensor.included_test", True),
FilterTest("climate.included_test", False),
FilterTest("climate.included_test", True),
]
for test in tests:

View File

@ -753,7 +753,7 @@ async def test_fetch_period_api_with_entity_glob_include_and_exclude(
{
"history": {
"exclude": {
"entity_globs": ["light.many*"],
"entity_globs": ["light.many*", "binary_sensor.*"],
},
"include": {
"entity_globs": ["light.m*"],
@ -769,6 +769,7 @@ async def test_fetch_period_api_with_entity_glob_include_and_exclude(
hass.states.async_set("light.many_state_changes", "on")
hass.states.async_set("switch.match", "on")
hass.states.async_set("media_player.test", "on")
hass.states.async_set("binary_sensor.exclude", "on")
await async_wait_recording_done(hass)
@ -778,10 +779,11 @@ async def test_fetch_period_api_with_entity_glob_include_and_exclude(
)
assert response.status == HTTPStatus.OK
response_json = await response.json()
assert len(response_json) == 3
assert response_json[0][0]["entity_id"] == "light.match"
assert response_json[1][0]["entity_id"] == "media_player.test"
assert response_json[2][0]["entity_id"] == "switch.match"
assert len(response_json) == 4
assert response_json[0][0]["entity_id"] == "light.many_state_changes"
assert response_json[1][0]["entity_id"] == "light.match"
assert response_json[2][0]["entity_id"] == "media_player.test"
assert response_json[3][0]["entity_id"] == "switch.match"
async def test_entity_ids_limit_via_api(hass, hass_client, recorder_mock):

View File

@ -866,7 +866,7 @@ async def test_event_listener_filtered_allowlist(
FilterTest("fake.excluded", False),
FilterTest("another_fake.denied", False),
FilterTest("fake.excluded_entity", False),
FilterTest("another_fake.included_entity", False),
FilterTest("another_fake.included_entity", True),
]
execute_filter_test(hass, tests, handler_method, write_api, get_mock_call)

View File

@ -2153,7 +2153,7 @@ async def test_include_exclude_events_with_glob_filters(
client = await hass_client()
entries = await _async_fetch_logbook(client)
assert len(entries) == 6
assert len(entries) == 7
_assert_entry(
entries[0], name="Home Assistant", message="started", domain=ha.DOMAIN
)
@ -2162,6 +2162,7 @@ async def test_include_exclude_events_with_glob_filters(
_assert_entry(entries[3], name="bla", entity_id=entity_id, state="20")
_assert_entry(entries[4], name="blu", entity_id=entity_id2, state="20")
_assert_entry(entries[5], name="included", entity_id=entity_id4, state="30")
_assert_entry(entries[6], name="included", entity_id=entity_id5, state="30")
async def test_empty_config(hass, hass_client, recorder_mock):

View File

@ -514,3 +514,128 @@ async def test_same_entity_included_excluded_include_domain_wins(hass, recorder_
assert filtered_events_entity_ids == filter_accept
assert not filtered_events_entity_ids.intersection(filter_reject)
async def test_specificly_included_entity_always_wins(hass, recorder_mock):
"""Test specificlly included entity always wins."""
filter_accept = {
"media_player.test2",
"media_player.test3",
"thermostat.test",
"binary_sensor.specific_include",
}
filter_reject = {
"binary_sensor.test2",
"binary_sensor.home",
"binary_sensor.can_cancel_this_one",
}
conf = {
CONF_INCLUDE: {
CONF_ENTITIES: ["binary_sensor.specific_include"],
},
CONF_EXCLUDE: {
CONF_DOMAINS: ["binary_sensor"],
CONF_ENTITY_GLOBS: ["binary_sensor.*"],
},
}
extracted_filter = extract_include_exclude_filter_conf(conf)
entity_filter = convert_include_exclude_filter(extracted_filter)
sqlalchemy_filter = sqlalchemy_filter_from_include_exclude_conf(extracted_filter)
assert sqlalchemy_filter is not None
for entity_id in filter_accept:
assert entity_filter(entity_id) is True
for entity_id in filter_reject:
assert entity_filter(entity_id) is False
(
filtered_states_entity_ids,
filtered_events_entity_ids,
) = await _async_get_states_and_events_with_filter(
hass, sqlalchemy_filter, filter_accept | filter_reject
)
assert filtered_states_entity_ids == filter_accept
assert not filtered_states_entity_ids.intersection(filter_reject)
assert filtered_events_entity_ids == filter_accept
assert not filtered_events_entity_ids.intersection(filter_reject)
async def test_specificly_included_entity_always_wins_over_glob(hass, recorder_mock):
"""Test specificlly included entity always wins over a glob."""
filter_accept = {
"sensor.apc900va_status",
"sensor.apc900va_battery_charge",
"sensor.apc900va_battery_runtime",
"sensor.apc900va_load",
"sensor.energy_x",
}
filter_reject = {
"sensor.apc900va_not_included",
}
conf = {
CONF_EXCLUDE: {
CONF_DOMAINS: [
"updater",
"camera",
"group",
"media_player",
"script",
"sun",
"automation",
"zone",
"weblink",
"scene",
"calendar",
"weather",
"remote",
"notify",
"switch",
"shell_command",
"media_player",
],
CONF_ENTITY_GLOBS: ["sensor.apc900va_*"],
},
CONF_INCLUDE: {
CONF_DOMAINS: [
"binary_sensor",
"climate",
"device_tracker",
"input_boolean",
"sensor",
],
CONF_ENTITY_GLOBS: ["sensor.energy_*"],
CONF_ENTITIES: [
"sensor.apc900va_status",
"sensor.apc900va_battery_charge",
"sensor.apc900va_battery_runtime",
"sensor.apc900va_load",
],
},
}
extracted_filter = extract_include_exclude_filter_conf(conf)
entity_filter = convert_include_exclude_filter(extracted_filter)
sqlalchemy_filter = sqlalchemy_filter_from_include_exclude_conf(extracted_filter)
assert sqlalchemy_filter is not None
for entity_id in filter_accept:
assert entity_filter(entity_id) is True
for entity_id in filter_reject:
assert entity_filter(entity_id) is False
(
filtered_states_entity_ids,
filtered_events_entity_ids,
) = await _async_get_states_and_events_with_filter(
hass, sqlalchemy_filter, filter_accept | filter_reject
)
assert filtered_states_entity_ids == filter_accept
assert not filtered_states_entity_ids.intersection(filter_reject)
assert filtered_events_entity_ids == filter_accept
assert not filtered_events_entity_ids.intersection(filter_reject)

View File

@ -91,8 +91,8 @@ def test_excludes_only_with_glob_case_3():
assert testfilter("cover.garage_door")
def test_with_include_domain_case4a():
"""Test case 4a - include and exclude specified, with included domain."""
def test_with_include_domain_case4():
"""Test case 4 - include and exclude specified, with included domain."""
incl_dom = {"light", "sensor"}
incl_ent = {"binary_sensor.working"}
excl_dom = {}
@ -108,8 +108,30 @@ def test_with_include_domain_case4a():
assert testfilter("sun.sun") is False
def test_with_include_glob_case4a():
"""Test case 4a - include and exclude specified, with included glob."""
def test_with_include_domain_exclude_glob_case4():
"""Test case 4 - include and exclude specified, with included domain but excluded by glob."""
incl_dom = {"light", "sensor"}
incl_ent = {"binary_sensor.working"}
incl_glob = {}
excl_dom = {}
excl_ent = {"light.ignoreme", "sensor.notworking"}
excl_glob = {"sensor.busted"}
testfilter = generate_filter(
incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob
)
assert testfilter("sensor.test")
assert testfilter("sensor.busted") is False
assert testfilter("sensor.notworking") is False
assert testfilter("light.test")
assert testfilter("light.ignoreme") is False
assert testfilter("binary_sensor.working")
assert testfilter("binary_sensor.another") is False
assert testfilter("sun.sun") is False
def test_with_include_glob_case4():
"""Test case 4 - include and exclude specified, with included glob."""
incl_dom = {}
incl_glob = {"light.*", "sensor.*"}
incl_ent = {"binary_sensor.working"}
@ -129,8 +151,8 @@ def test_with_include_glob_case4a():
assert testfilter("sun.sun") is False
def test_with_include_domain_glob_filtering_case4a():
"""Test case 4a - include and exclude specified, both have domains and globs."""
def test_with_include_domain_glob_filtering_case4():
"""Test case 4 - include and exclude specified, both have domains and globs."""
incl_dom = {"light"}
incl_glob = {"*working"}
incl_ent = {}
@ -142,17 +164,64 @@ def test_with_include_domain_glob_filtering_case4a():
)
assert testfilter("sensor.working")
assert testfilter("sensor.notworking") is False
assert testfilter("sensor.notworking") is True # include is stronger
assert testfilter("light.test")
assert testfilter("light.notworking") is False
assert testfilter("light.notworking") is True # include is stronger
assert testfilter("light.ignoreme") is False
assert testfilter("binary_sensor.not_working") is False
assert testfilter("binary_sensor.not_working") is True # include is stronger
assert testfilter("binary_sensor.another") is False
assert testfilter("sun.sun") is False
def test_exclude_domain_case4b():
"""Test case 4b - include and exclude specified, with excluded domain."""
def test_with_include_domain_glob_filtering_case4a_include_strong():
"""Test case 4 - include and exclude specified, both have domains and globs, and a specifically included entity."""
incl_dom = {"light"}
incl_glob = {"*working"}
incl_ent = {"binary_sensor.specificly_included"}
excl_dom = {"binary_sensor"}
excl_glob = {"*notworking"}
excl_ent = {"light.ignoreme"}
testfilter = generate_filter(
incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob
)
assert testfilter("sensor.working")
assert testfilter("sensor.notworking") is True # iclude is stronger
assert testfilter("light.test")
assert testfilter("light.notworking") is True # iclude is stronger
assert testfilter("light.ignoreme") is False
assert testfilter("binary_sensor.not_working") is True # iclude is stronger
assert testfilter("binary_sensor.another") is False
assert testfilter("binary_sensor.specificly_included") is True
assert testfilter("sun.sun") is False
def test_with_include_glob_filtering_case4a_include_strong():
"""Test case 4 - include and exclude specified, both have globs, and a specifically included entity."""
incl_dom = {}
incl_glob = {"*working"}
incl_ent = {"binary_sensor.specificly_included"}
excl_dom = {}
excl_glob = {"*broken", "*notworking", "binary_sensor.*"}
excl_ent = {"light.ignoreme"}
testfilter = generate_filter(
incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob
)
assert testfilter("sensor.working") is True
assert testfilter("sensor.notworking") is True # include is stronger
assert testfilter("sensor.broken") is False
assert testfilter("light.test") is False
assert testfilter("light.notworking") is True # include is stronger
assert testfilter("light.ignoreme") is False
assert testfilter("binary_sensor.not_working") is True # include is stronger
assert testfilter("binary_sensor.another") is False
assert testfilter("binary_sensor.specificly_included") is True
assert testfilter("sun.sun") is False
def test_exclude_domain_case5():
"""Test case 5 - include and exclude specified, with excluded domain."""
incl_dom = {}
incl_ent = {"binary_sensor.working"}
excl_dom = {"binary_sensor"}
@ -168,8 +237,8 @@ def test_exclude_domain_case4b():
assert testfilter("sun.sun") is True
def test_exclude_glob_case4b():
"""Test case 4b - include and exclude specified, with excluded glob."""
def test_exclude_glob_case5():
"""Test case 5 - include and exclude specified, with excluded glob."""
incl_dom = {}
incl_glob = {}
incl_ent = {"binary_sensor.working"}
@ -189,8 +258,29 @@ def test_exclude_glob_case4b():
assert testfilter("sun.sun") is True
def test_no_domain_case4c():
"""Test case 4c - include and exclude specified, with no domains."""
def test_exclude_glob_case5_include_strong():
"""Test case 5 - include and exclude specified, with excluded glob, and a specifically included entity."""
incl_dom = {}
incl_glob = {}
incl_ent = {"binary_sensor.working"}
excl_dom = {"binary_sensor"}
excl_glob = {"binary_sensor.*"}
excl_ent = {"light.ignoreme", "sensor.notworking"}
testfilter = generate_filter(
incl_dom, incl_ent, excl_dom, excl_ent, incl_glob, excl_glob
)
assert testfilter("sensor.test")
assert testfilter("sensor.notworking") is False
assert testfilter("light.test")
assert testfilter("light.ignoreme") is False
assert testfilter("binary_sensor.working")
assert testfilter("binary_sensor.another") is False
assert testfilter("sun.sun") is True
def test_no_domain_case6():
"""Test case 6 - include and exclude specified, with no domains."""
incl_dom = {}
incl_ent = {"binary_sensor.working"}
excl_dom = {}