Improve coverage reports on codecov (#3200)

* ci.coverage: enable branch coverage

* ci.coverage: gather coverage from tests as well

* ci.codecov: turn codecov config into a dotfile

* ci.codecov: disable comment, split reports by path

* ci.codecov: require all reports before notifying

.codecov.yml

@@ -0,0 +1,44 @@
+codecov:
+  notify:
+    require_ci_to_pass: true
+    # wait until at all test runners have uploaded a report (see the test job's build matrix)
+    # otherwise, coverage failures may be shown while some reports are still missing
+    after_n_builds: 10
+comment:
+  # this also configures the layout of PR check summaries / comments
+  layout: "reach, diff, flags, files"
+  # don't ever let the codecov bot comment on PRs
+  after_n_builds: 999999999
+coverage:
+  range: "50...100"
+  precision: 2
+  round: down
+  status:
+    changes: false
+    patch: false
+    # split up coverage reports by path
+    # don't set coverage targets and instead set coverage thresholds
+    project:
+      # we can't disable the overall default project, because only this will have PR check summaries / comments
+      # this replaces the PR comment
+      default:
+        target: 50
+      streamlink:
+        threshold: 1
+        paths:
+          - "src/streamlink/"
+          - "!src/streamlink/plugins/"
+      streamlink_cli:
+        threshold: 1
+        paths:
+          - "src/streamlink_cli/"
+      plugins:
+        # don't set a threshold on plugins
+        target: 30
+        paths:
+          - "src/streamlink/plugins/"
+      tests:
+        # new tests should always be fully covered
+        threshold: 0
+        paths:
+          - "tests/"

.coveragerc

@@ -1,6 +1,7 @@
 [run]
 source =
     src
+    tests
 
 [report]
 omit =

.github/workflows/main.yml

@@ -17,7 +17,8 @@ jobs:
     name: Test
     strategy:
       fail-fast: false
-      # when changing the build matrix, the `after_n_builds` value in codecov.yml may need to be updated
+      # please remember to change the `codecov.notify.after_n_builds` value in .codecov.yml
+      # when changing the build matrix and changing the number of test runners
       matrix:
         os: [ubuntu-latest, windows-latest]
         python: [2.7, 3.5, 3.6, 3.7, 3.8]
@@ -36,7 +37,7 @@ jobs:
         run: bash ./script/install-dependencies.sh
       - name: Test
         continue-on-error: ${{ matrix.continue || false }}
-        run: pytest -r a --cov --cov-report=xml
+        run: pytest -r a --cov --cov-branch --cov-report=xml
       - name: Lint
         continue-on-error: ${{ matrix.continue || false }}
         run: flake8

codecov.yml

@@ -1,21 +0,0 @@
-codecov:
-  notify:
-    require_ci_to_pass: true
-    # wait until at least one linux and one windows build has succeeded (see the test job's build matrix)
-    after_n_builds: 6
-comment:
-  behavior: default
-  layout: header, diff
-  require_changes: false
-coverage:
-  precision: 2
-  range:
-  - 50.0
-  - 100.0
-  round: down
-  status:
-    changes: false
-    patch: false
-    project:
-      default:
-        target: 30