github
/
ha-core
mirror of https://github.com/home-assistant/core
1
Fork 0
Code Releases Activity

Generate requirements per supported architecture (#115708) 

* Generate requirements per supported architecture

* Don't store wheels requirements in the repo

* Dry run

* Set Python version

* Install base packages

* Fix

* Fix

* Fix

* Fix typo

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

* Genarate requirements_all_pytest.txt

* Fix hassfest

* Reenable building wheels

* Remove unneeded code

* Address review comment

* Fix lying comment

* Add tests, address review comments

* Deduplicate

* Fix file name

* Add comment

---------

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>
This commit is contained in:
Erik Montnemery 2024-04-22 19:23:08 +02:00 committed by GitHub
parent 124eca4d53
commit 2caca7fbe3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 165 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/ci.yaml vendored
View File

@ -97,7 +97,8 @@ jobs:
hashFiles('requirements_test.txt', 'requirements_test_pre_commit.txt') }}-${{
hashFiles('requirements.txt') }}-${{
hashFiles('requirements_all.txt') }}-${{
hashFiles('homeassistant/package_constraints.txt') }}" >> $GITHUB_OUTPUT
hashFiles('homeassistant/package_constraints.txt') }}-${{
hashFiles('script/gen_requirements_all.py') }}" >> $GITHUB_OUTPUT
- name: Generate partial pre-commit restore key
id: generate_pre-commit_cache_key
run: >-
@ -497,8 +498,9 @@ jobs:
python --version
pip install "$(grep '^uv' < requirements_test.txt)"
uv pip install -U "pip>=21.3.1" setuptools wheel
uv pip install -r requirements_all.txt
uv pip install "$(grep 'python-gammu' < requirements_all.txt | sed -e 's|# python-gammu|python-gammu|g')"
uv pip install -r requirements.txt
python -m script.gen_requirements_all ci
uv pip install -r requirements_all_pytest.txt
uv pip install -r requirements_test.txt
uv pip install -e . --config-settings editable_mode=compat

60
.github/workflows/wheels.yml vendored
View File

@ -14,6 +14,10 @@ on:
- "homeassistant/package_constraints.txt"
- "requirements_all.txt"
- "requirements.txt"
- "script/gen_requirements_all.py"
env:
DEFAULT_PYTHON: "3.12"
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name}}
@ -30,6 +34,21 @@ jobs:
- name: Checkout the repository
uses: actions/checkout@v4.1.3
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/setup-python@v5.1.0
with:
python-version: ${{ env.DEFAULT_PYTHON }}
check-latest: true
- name: Create Python virtual environment
run: |
python -m venv venv
. venv/bin/activate
python --version
pip install "$(grep '^uv' < requirements_test.txt)"
uv pip install -r requirements.txt
- name: Get information
id: info
uses: home-assistant/actions/helpers/info@master
@ -76,6 +95,17 @@ jobs:
path: ./requirements_diff.txt
overwrite: true
- name: Generate requirements
run: |
. venv/bin/activate
python -m script.gen_requirements_all ci
- name: Upload requirements_all_wheels
uses: actions/upload-artifact@v4.3.1
with:
name: requirements_all_wheels
path: ./requirements_all_wheels_*.txt
core:
name: Build Core wheels ${{ matrix.abi }} for ${{ matrix.arch }} (musllinux_1_2)
if: github.repository_owner == 'home-assistant'
@ -138,30 +168,10 @@ jobs:
with:
name: requirements_diff
- name: (Un)comment packages
run: |
requirement_files="requirements_all.txt requirements_diff.txt"
for requirement_file in ${requirement_files}; do
sed -i "s|# pyuserinput|pyuserinput|g" ${requirement_file}
sed -i "s|# evdev|evdev|g" ${requirement_file}
sed -i "s|# pycups|pycups|g" ${requirement_file}
sed -i "s|# decora-wifi|decora-wifi|g" ${requirement_file}
sed -i "s|# python-gammu|python-gammu|g" ${requirement_file}
# Some packages are not buildable on armhf anymore
if [ "${{ matrix.arch }}" = "armhf" ]; then
# Pandas has issues building on armhf, it is expected they
# will drop the platform in the near future (they consider it
# "flimsy" on 386). The following packages depend on pandas,
# so we comment them out.
sed -i "s|env-canada|# env-canada|g" ${requirement_file}
sed -i "s|noaa-coops|# noaa-coops|g" ${requirement_file}
sed -i "s|pyezviz|# pyezviz|g" ${requirement_file}
sed -i "s|pykrakenapi|# pykrakenapi|g" ${requirement_file}
fi
done
- name: Download requirements_all_wheels
uses: actions/download-artifact@v4.1.4
with:
name: requirements_all_wheels
- name: Split requirements all
run: |
@ -169,7 +179,7 @@ jobs:
# This is to prevent the build from running out of memory when
# resolving packages on 32-bits systems (like armhf, armv7).
split -l $(expr $(expr $(cat requirements_all.txt | wc -l) + 1) / 3) requirements_all.txt requirements_all.txt
split -l $(expr $(expr $(cat requirements_all.txt | wc -l) + 1) / 3) requirements_all_wheels_${{ matrix.arch }}.txt requirements_all.txt
- name: Create requirements for cython<3
run: |

104
script/gen_requirements_all.py
View File

@ -17,7 +17,10 @@ from typing import Any
from homeassistant.util.yaml.loader import load_yaml
from script.hassfest.model import Integration
COMMENT_REQUIREMENTS = (
# Requirements which can't be installed on all systems because they rely on additional
# system packages. Requirements listed in EXCLUDED_REQUIREMENTS_ALL will be commented-out
# in requirements_all.txt and requirements_test_all.txt.
EXCLUDED_REQUIREMENTS_ALL = {
"atenpdu", # depends on pysnmp which is not maintained at this time
"avea", # depends on bluepy
"avion",
@ -36,10 +39,39 @@ COMMENT_REQUIREMENTS = (
"pyuserinput",
"tensorflow",
"tf-models-official",
)
}
COMMENT_REQUIREMENTS_NORMALIZED = {
commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS
# Requirements excluded by EXCLUDED_REQUIREMENTS_ALL which should be included when
# building integration wheels for all architectures.
INCLUDED_REQUIREMENTS_WHEELS = {
"decora-wifi",
"evdev",
"pycups",
"python-gammu",
"pyuserinput",
}
# Requirements to exclude or include when running github actions.
# Requirements listed in "exclude" will be commented-out in
# requirements_all_{action}.txt
# Requirements listed in "include" must be listed in EXCLUDED_REQUIREMENTS_CI, and
# will be included in requirements_all_{action}.txt
OVERRIDDEN_REQUIREMENTS_ACTIONS = {
"pytest": {"exclude": set(), "include": {"python-gammu"}},
"wheels_aarch64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
# Pandas has issues building on armhf, it is expected they
# will drop the platform in the near future (they consider it
# "flimsy" on 386). The following packages depend on pandas,
# so we comment them out.
"wheels_armhf": {
"exclude": {"env-canada", "noaa-coops", "pyezviz", "pykrakenapi"},
"include": INCLUDED_REQUIREMENTS_WHEELS,
},
"wheels_armv7": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
"wheels_amd64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
"wheels_i386": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
}
IGNORE_PIN = ("colorlog>2.1,<3", "urllib3")
@ -254,6 +286,12 @@ def gather_recursive_requirements(
return reqs
def _normalize_package_name(package_name: str) -> str:
"""Normalize a package name."""
# pipdeptree needs lowercase and dash instead of underscore or period as separator
return package_name.lower().replace("_", "-").replace(".", "-")
def normalize_package_name(requirement: str) -> str:
"""Return a normalized package name from a requirement string."""
# This function is also used in hassfest.
@ -262,12 +300,24 @@ def normalize_package_name(requirement: str) -> str:
return ""
# pipdeptree needs lowercase and dash instead of underscore or period as separator
return match.group(1).lower().replace("_", "-").replace(".", "-")
return _normalize_package_name(match.group(1))
def comment_requirement(req: str) -> bool:
"""Comment out requirement. Some don't install on all systems."""
return normalize_package_name(req) in COMMENT_REQUIREMENTS_NORMALIZED
return normalize_package_name(req) in EXCLUDED_REQUIREMENTS_ALL
def process_action_requirement(req: str, action: str) -> str:
"""Process requirement for a specific github action."""
normalized_package_name = normalize_package_name(req)
if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["exclude"]:
return f"# {req}"
if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["include"]:
return req
if normalized_package_name in EXCLUDED_REQUIREMENTS_ALL:
return f"# {req}"
return req
def gather_modules() -> dict[str, list[str]] | None:
@ -353,6 +403,16 @@ def generate_requirements_list(reqs: dict[str, list[str]]) -> str:
return "".join(output)
def generate_action_requirements_list(reqs: dict[str, list[str]], action: str) -> str:
"""Generate a pip file based on requirements."""
output = []
for pkg, requirements in sorted(reqs.items(), key=itemgetter(0)):
output.extend(f"\n# {req}" for req in sorted(requirements))
processed_pkg = process_action_requirement(pkg, action)
output.append(f"\n{processed_pkg}\n")
return "".join(output)
def requirements_output() -> str:
"""Generate output for requirements."""
output = [
@ -379,6 +439,18 @@ def requirements_all_output(reqs: dict[str, list[str]]) -> str:
return "".join(output)
def requirements_all_action_output(reqs: dict[str, list[str]], action: str) -> str:
"""Generate output for requirements_all_{action}."""
output = [
f"# Home Assistant Core, full dependency set for {action}\n",
GENERATED_MESSAGE,
"-r requirements.txt\n",
]
output.append(generate_action_requirements_list(reqs, action))
return "".join(output)
def requirements_test_all_output(reqs: dict[str, list[str]]) -> str:
"""Generate output for test_requirements."""
output = [
@ -459,7 +531,7 @@ def diff_file(filename: str, content: str) -> list[str]:
)
def main(validate: bool) -> int:
def main(validate: bool, ci: bool) -> int:
"""Run the script."""
if not os.path.isfile("requirements_all.txt"):
print("Run this from HA root dir")
@ -472,17 +544,28 @@ def main(validate: bool) -> int:
reqs_file = requirements_output()
reqs_all_file = requirements_all_output(data)
reqs_all_action_files = {
action: requirements_all_action_output(data, action)
for action in OVERRIDDEN_REQUIREMENTS_ACTIONS
}
reqs_test_all_file = requirements_test_all_output(data)
# Always calling requirements_pre_commit_output is intentional to ensure
# the code is called by the pre-commit hooks.
reqs_pre_commit_file = requirements_pre_commit_output()
constraints = gather_constraints()
files = (
files = [
("requirements.txt", reqs_file),
("requirements_all.txt", reqs_all_file),
("requirements_test_pre_commit.txt", reqs_pre_commit_file),
("requirements_test_all.txt", reqs_test_all_file),
("homeassistant/package_constraints.txt", constraints),
)
]
if ci:
files.extend(
(f"requirements_all_{action}.txt", reqs_all_file)
for action, reqs_all_file in reqs_all_action_files.items()
)
if validate:
errors = []
@ -511,4 +594,5 @@ def main(validate: bool) -> int:
if __name__ == "__main__":
_VAL = sys.argv[-1] == "validate"
sys.exit(main(_VAL))
_CI = sys.argv[-1] == "ci"
sys.exit(main(_VAL, _CI))

10
script/hassfest/requirements.py
View File

@ -15,13 +15,13 @@ from awesomeversion import AwesomeVersion, AwesomeVersionStrategy
from tqdm import tqdm
import homeassistant.util.package as pkg_util
from script.gen_requirements_all import COMMENT_REQUIREMENTS, normalize_package_name
from script.gen_requirements_all import (
EXCLUDED_REQUIREMENTS_ALL,
normalize_package_name,
)
from .model import Config, Integration
IGNORE_PACKAGES = {
commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS
}
PACKAGE_REGEX = re.compile(
r"^(?:--.+\s)?([-_,\.\w\d\[\]]+)(==|>=|<=|~=|!=|<|>|===)*(.*)$"
)
@ -116,7 +116,7 @@ def validate_requirements(integration: Integration) -> None:
f"Failed to normalize package name from requirement {req}",
)
return
if package in IGNORE_PACKAGES:
if package in EXCLUDED_REQUIREMENTS_ALL:
continue
integration_requirements.add(req)
integration_packages.add(package)

1
tests/script/__init__.py Normal file
View File

@ -0,0 +1 @@
"""Tests for scripts."""

25
tests/script/test_gen_requirements_all.py Normal file
View File

@ -0,0 +1,25 @@
"""Tests for the gen_requirements_all script."""
from script import gen_requirements_all
def test_overrides_normalized() -> None:
"""Test override lists are using normalized package names."""
for req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL:
assert req == gen_requirements_all._normalize_package_name(req)
for req in gen_requirements_all.INCLUDED_REQUIREMENTS_WHEELS:
assert req == gen_requirements_all._normalize_package_name(req)
for overrides in gen_requirements_all.OVERRIDDEN_REQUIREMENTS_ACTIONS.values():
for req in overrides["exclude"]:
assert req == gen_requirements_all._normalize_package_name(req)
for req in overrides["include"]:
assert req == gen_requirements_all._normalize_package_name(req)
def test_include_overrides_subsets() -> None:
"""Test packages in include override lists are present in the exclude list."""
for req in gen_requirements_all.INCLUDED_REQUIREMENTS_WHEELS:
assert req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL
for overrides in gen_requirements_all.OVERRIDDEN_REQUIREMENTS_ACTIONS.values():
for req in overrides["include"]:
assert req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL