From c76c73b05022675246748b84dd0f8bb3e751d3b9 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Fri, 11 Oct 2024 16:20:52 -0400 Subject: [PATCH 1/4] Adding basic network upload command --- codecov_cli/commands/network_upload.py | 133 +++++++++++++++++++ codecov_cli/main.py | 3 +- codecov_cli/services/upload/upload_sender.py | 11 +- 3 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 codecov_cli/commands/network_upload.py diff --git a/codecov_cli/commands/network_upload.py b/codecov_cli/commands/network_upload.py new file mode 100644 index 000000000..2880f71a9 --- /dev/null +++ b/codecov_cli/commands/network_upload.py @@ -0,0 +1,133 @@ +import logging +import pathlib +import typing + +import click + +from codecov_cli.services.upload.network_finder import NetworkFinder +from codecov_cli.services.upload.upload_sender import UploadSender +from codecov_cli.helpers.options import global_options +from codecov_cli.types import CommandContext, UploadCollectionResult +from codecov_cli.helpers.request import log_warnings_and_errors_if_any + +logger = logging.getLogger("codecovcli") + +@click.command() +@click.option( + "--root-dir", + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=pathlib.Path), + default=pathlib.Path.cwd(), + help="Root directory for searching files (default: current working directory)", +) +@click.option( + "--network-filter", + help="Regex to filter files (e.g., '.*\\.py')", +) +@click.option( + "--network-prefix", + help="Prefix to prepend to file paths", +) +@click.option( + "--include-git-files", + is_flag=True, + default=False, + help="Include files tracked by git", +) +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Perform a dry run without actually uploading files", +) +@click.option( + "--commit-sha", + required=True, + help="Commit SHA to associate with the upload", +) +@click.option( + "--token", + required=True, + help="Codecov token for authentication", +) +@global_options +@click.pass_context +def network_upload( + ctx: CommandContext, + root_dir: pathlib.Path, + network_filter: typing.Optional[str], + network_prefix: typing.Optional[str], + include_git_files: bool, + dry_run: bool, + commit_sha: str, + slug: typing.Optional[str], + token: typing.Optional[str], + git_service: typing.Optional[str], + fail_on_error: bool, +): + """ + Find and list files in the project network, then upload them to Codecov. + """ + network_finder = NetworkFinder( + versioning_system=ctx.obj["versioning_system"], + network_root_folder=root_dir, + network_filter=network_filter, + network_prefix=network_prefix, + ) + + files = network_finder.find_files(include_git_files) + + if not files: + logger.warning("No files found in the network.") + return + + logger.info(f"Found {len(files)} files in the network:") + + + if dry_run: + logger.info("Dry run: No files will be uploaded.") + for file in files: + click.echo(file) + return + + logger.info("Preparing to upload files...") + + # Prepare the upload data + upload_data = UploadCollectionResult( + network=files, + files=[], # We're not uploading coverage files in this command + file_fixes=[], + ) + + # Create an UploadSender instance + sender = UploadSender() + + # Send the upload data + sending_result = sender.send_upload_data( + upload_data, + commit_sha, + token, + env_vars={}, + report_code=None, + upload_file_type="network", + name=None, + branch=None, + slug=slug, + pull_request_number=None, + build_code=None, + build_url=None, + job_code=None, + flags=[], + ci_service=None, + git_service=git_service, + enterprise_url=ctx.obj.get("enterprise_url"), + ) + + # Log any warnings or errors + log_warnings_and_errors_if_any(sending_result, "Network Upload", fail_on_error=False) + + if sending_result.status_code == 200: + logger.info("Network files successfully uploaded to Codecov.") + else: + logger.error(f"Failed to upload network files. Status code: {sending_result.status_code}") + if fail_on_error: + exit(1) diff --git a/codecov_cli/main.py b/codecov_cli/main.py index 9505aaa63..7922c8b45 100644 --- a/codecov_cli/main.py +++ b/codecov_cli/main.py @@ -17,6 +17,7 @@ from codecov_cli.commands.staticanalysis import static_analysis from codecov_cli.commands.upload import do_upload from codecov_cli.commands.upload_process import upload_process +from codecov_cli.commands.network_upload import network_upload from codecov_cli.helpers.ci_adapters import get_ci_adapter, get_ci_providers_list from codecov_cli.helpers.config import load_cli_config from codecov_cli.helpers.logging_utils import configure_logger @@ -77,7 +78,7 @@ def cli( cli.add_command(upload_process) cli.add_command(send_notifications) cli.add_command(process_test_results) - +cli.add_command(network_upload) def run(): cli(obj={}) diff --git a/codecov_cli/services/upload/upload_sender.py b/codecov_cli/services/upload/upload_sender.py index 22f8924ad..c634df5f8 100644 --- a/codecov_cli/services/upload/upload_sender.py +++ b/codecov_cli/services/upload/upload_sender.py @@ -115,6 +115,10 @@ def _generate_payload( payload = { "test_results_files": self._get_files(upload_data), } + elif upload_file_type == "network": + payload = { + "network_files": network_files if network_files is not None else [], + } json_data = json.dumps(payload) return json_data.encode() @@ -185,5 +189,10 @@ def get_url_and_possibly_update_data( data["commit"] = commit_sha data["service"] = git_service url = f"{upload_url}/upload/test_results/v1" - + elif report_type == "network": + data["slug"] = encoded_slug + data["branch"] = branch + data["commit"] = commit_sha + data["service"] = git_service + url = f"{upload_url}/upload/network/v1" return url, data From d4bdd7ae75ca4ee8d6e77caa54e079f4a8d35eb9 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Fri, 11 Oct 2024 19:25:38 -0400 Subject: [PATCH 2/4] lint --- codecov_cli/commands/network_upload.py | 20 +++++++++++++------- codecov_cli/main.py | 3 ++- tests/commands/test_process_test_results.py | 1 + 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/codecov_cli/commands/network_upload.py b/codecov_cli/commands/network_upload.py index 2880f71a9..0d8ec6bbf 100644 --- a/codecov_cli/commands/network_upload.py +++ b/codecov_cli/commands/network_upload.py @@ -4,18 +4,21 @@ import click +from codecov_cli.helpers.options import global_options +from codecov_cli.helpers.request import log_warnings_and_errors_if_any from codecov_cli.services.upload.network_finder import NetworkFinder from codecov_cli.services.upload.upload_sender import UploadSender -from codecov_cli.helpers.options import global_options from codecov_cli.types import CommandContext, UploadCollectionResult -from codecov_cli.helpers.request import log_warnings_and_errors_if_any logger = logging.getLogger("codecovcli") + @click.command() @click.option( "--root-dir", - type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=pathlib.Path), + type=click.Path( + exists=True, file_okay=False, dir_okay=True, path_type=pathlib.Path + ), default=pathlib.Path.cwd(), help="Root directory for searching files (default: current working directory)", ) @@ -81,7 +84,6 @@ def network_upload( return logger.info(f"Found {len(files)} files in the network:") - if dry_run: logger.info("Dry run: No files will be uploaded.") @@ -123,11 +125,15 @@ def network_upload( ) # Log any warnings or errors - log_warnings_and_errors_if_any(sending_result, "Network Upload", fail_on_error=False) + log_warnings_and_errors_if_any( + sending_result, "Network Upload", fail_on_error=False + ) if sending_result.status_code == 200: logger.info("Network files successfully uploaded to Codecov.") else: - logger.error(f"Failed to upload network files. Status code: {sending_result.status_code}") + logger.error( + f"Failed to upload network files. Status code: {sending_result.status_code}" + ) if fail_on_error: - exit(1) + exit(1) diff --git a/codecov_cli/main.py b/codecov_cli/main.py index 7922c8b45..bf0aa619a 100644 --- a/codecov_cli/main.py +++ b/codecov_cli/main.py @@ -11,13 +11,13 @@ from codecov_cli.commands.empty_upload import empty_upload from codecov_cli.commands.get_report_results import get_report_results from codecov_cli.commands.labelanalysis import label_analysis +from codecov_cli.commands.network_upload import network_upload from codecov_cli.commands.process_test_results import process_test_results from codecov_cli.commands.report import create_report from codecov_cli.commands.send_notifications import send_notifications from codecov_cli.commands.staticanalysis import static_analysis from codecov_cli.commands.upload import do_upload from codecov_cli.commands.upload_process import upload_process -from codecov_cli.commands.network_upload import network_upload from codecov_cli.helpers.ci_adapters import get_ci_adapter, get_ci_providers_list from codecov_cli.helpers.config import load_cli_config from codecov_cli.helpers.logging_utils import configure_logger @@ -80,6 +80,7 @@ def cli( cli.add_command(process_test_results) cli.add_command(network_upload) + def run(): cli(obj={}) diff --git a/tests/commands/test_process_test_results.py b/tests/commands/test_process_test_results.py index e583159dc..9ef89adc0 100644 --- a/tests/commands/test_process_test_results.py +++ b/tests/commands/test_process_test_results.py @@ -43,6 +43,7 @@ def test_process_test_results( # Ensure that there's an output assert result.output + def test_process_test_results_create_github_message( mocker, tmpdir, From c2fcc36a19da77e550a703ac2be09637dde81bd0 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Fri, 11 Oct 2024 19:27:18 -0400 Subject: [PATCH 3/4] Fix test --- tests/test_codecov_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_codecov_cli.py b/tests/test_codecov_cli.py index 80136f06f..00f773dac 100644 --- a/tests/test_codecov_cli.py +++ b/tests/test_codecov_cli.py @@ -10,6 +10,7 @@ def test_existing_commands(): "empty-upload", "get-report-results", "label-analysis", + "network-upload", "pr-base-picking", "process-test-results", "send-notifications", From 8399c4097ad1685bf737ba0a88da6a72e736dad6 Mon Sep 17 00:00:00 2001 From: trent-codecov Date: Fri, 11 Oct 2024 19:30:28 -0400 Subject: [PATCH 4/4] Don't do full ci for the moment --- .github/workflows/ci.yml | 322 +++++++++++++++++++-------------------- 1 file changed, 161 insertions(+), 161 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e96fda3a..b244b43da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,170 +1,170 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# # This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: CLI CI +# name: CLI CI -on: - pull_request: - push: - branches: - - main +# on: +# pull_request: +# push: +# branches: +# - main -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install black==22.3.0 isort==5.10.1 - - name: Check linting with black - run: | - black --check codecov_cli - - name: Check imports order with isort - run: | - isort --check --profile=black codecov_cli -p staticcodecov_languages +# jobs: +# lint: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install black==22.3.0 isort==5.10.1 +# - name: Check linting with black +# run: | +# black --check codecov_cli +# - name: Check imports order with isort +# run: | +# isort --check --profile=black codecov_cli -p staticcodecov_languages - codecov-startup: - runs-on: ubuntu-latest - if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 2 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install CLI - run: | - pip install codecov-cli - - name: Create commit in codecov - run: | - codecovcli create-commit -t ${{ secrets.CODECOV_TOKEN }} --git-service github - - name: Create commit report in codecov - run: | - codecovcli create-report -t ${{ secrets.CODECOV_TOKEN }} --git-service github +# codecov-startup: +# runs-on: ubuntu-latest +# if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# fetch-depth: 2 +# - uses: actions/setup-python@v5 +# with: +# python-version: "3.12" +# - name: Install CLI +# run: | +# pip install codecov-cli +# - name: Create commit in codecov +# run: | +# codecovcli create-commit -t ${{ secrets.CODECOV_TOKEN }} --git-service github +# - name: Create commit report in codecov +# run: | +# codecovcli create-report -t ${{ secrets.CODECOV_TOKEN }} --git-service github - build-test-upload: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - python-version: "3.12" - - python-version: "3.11" - - python-version: "3.10" - - python-version: "3.9" - - python-version: "3.8" - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 2 - - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v5 - with: - python-version: "${{matrix.python-version}}" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - python -m pip install -e . - pip install -r tests/requirements.txt - - name: Test with pytest - run: | - pytest --cov --junitxml=${{matrix.python-version}}junit.xml - - name: Dogfooding codecov-cli - if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} - run: | - codecovcli -v do-upload --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --plugin pycoverage --flag python${{matrix.python-version}} - codecovcli do-upload --report-type test_results --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --plugin pycoverage --flag python${{matrix.python-version}} - - name: Upload artifacts for test-results-processing - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: ${{matrix.python-version}}junit.xml - path: ${{matrix.python-version}}junit.xml +# build-test-upload: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# include: +# - python-version: "3.12" +# - python-version: "3.11" +# - python-version: "3.10" +# - python-version: "3.9" +# - python-version: "3.8" +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# fetch-depth: 2 +# - name: Set up Python ${{matrix.python-version}} +# uses: actions/setup-python@v5 +# with: +# python-version: "${{matrix.python-version}}" +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install -r requirements.txt +# python -m pip install -e . +# pip install -r tests/requirements.txt +# - name: Test with pytest +# run: | +# pytest --cov --junitxml=${{matrix.python-version}}junit.xml +# - name: Dogfooding codecov-cli +# if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} +# run: | +# codecovcli -v do-upload --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --plugin pycoverage --flag python${{matrix.python-version}} +# codecovcli do-upload --report-type test_results --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --plugin pycoverage --flag python${{matrix.python-version}} +# - name: Upload artifacts for test-results-processing +# if: ${{ !cancelled() }} +# uses: actions/upload-artifact@v4 +# with: +# name: ${{matrix.python-version}}junit.xml +# path: ${{matrix.python-version}}junit.xml - static-analysis: - runs-on: ubuntu-latest - needs: codecov-startup - if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 2 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install CLI - run: | - pip install codecov-cli - - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 2 - - name: Static Analysis - run: | - codecovcli static-analysis --token ${{ secrets.STATIC_TOKEN }} +# static-analysis: +# runs-on: ubuntu-latest +# needs: codecov-startup +# if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# fetch-depth: 2 +# - uses: actions/setup-python@v5 +# with: +# python-version: "3.12" +# - name: Install CLI +# run: | +# pip install codecov-cli +# - uses: actions/checkout@v3 +# with: +# submodules: true +# fetch-depth: 2 +# - name: Static Analysis +# run: | +# codecovcli static-analysis --token ${{ secrets.STATIC_TOKEN }} - label-analysis: - runs-on: ubuntu-latest - needs: static-analysis - if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install dependencies for Dogfooding - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - python -m pip install -e . - pip install -r tests/requirements.txt - - name: Label Analysis - run: | - BASE_SHA=$(git merge-base HEAD^ origin/main) - echo $BASE_SHA - codecovcli label-analysis --token ${{ secrets.STATIC_TOKEN }} --base-sha=$BASE_SHA --max-wait-time=120 - - name: Upload smart-labels - run: | - codecovcli --codecov-yml-path=codecov.yml do-upload --plugin pycoverage --plugin compress-pycoverage --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --flag smart-labels +# label-analysis: +# runs-on: ubuntu-latest +# needs: static-analysis +# if: ${{ !github.event.pull_request.head.repo.fork && github.repository_owner == 'codecov' }} +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# fetch-depth: 0 +# - uses: actions/setup-python@v5 +# with: +# python-version: "3.12" +# - name: Install dependencies for Dogfooding +# run: | +# python -m pip install --upgrade pip +# pip install -r requirements.txt +# python -m pip install -e . +# pip install -r tests/requirements.txt +# - name: Label Analysis +# run: | +# BASE_SHA=$(git merge-base HEAD^ origin/main) +# echo $BASE_SHA +# codecovcli label-analysis --token ${{ secrets.STATIC_TOKEN }} --base-sha=$BASE_SHA --max-wait-time=120 +# - name: Upload smart-labels +# run: | +# codecovcli --codecov-yml-path=codecov.yml do-upload --plugin pycoverage --plugin compress-pycoverage --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --flag smart-labels - process-test-results: - if: ${{ always() }} - needs: build-test-upload - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 2 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install dependencies for Dogfooding - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - python -m pip install -e . - pip install -r tests/requirements.txt - - name: Download all test results - uses: actions/download-artifact@v4 - with: - pattern: "*junit.xml" - path: "test_results" - merge-multiple: true +# process-test-results: +# if: ${{ always() }} +# needs: build-test-upload +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# fetch-depth: 2 +# - uses: actions/setup-python@v5 +# with: +# python-version: "3.12" +# - name: Install dependencies for Dogfooding +# run: | +# python -m pip install --upgrade pip +# pip install -r requirements.txt +# python -m pip install -e . +# pip install -r tests/requirements.txt +# - name: Download all test results +# uses: actions/download-artifact@v4 +# with: +# pattern: "*junit.xml" +# path: "test_results" +# merge-multiple: true - - name: Dogfooding codecov-cli - if: ${{ !cancelled() && github.ref && contains(github.ref, 'pull') }} - run: | - codecovcli process-test-results --dir test_results --github-token ${{ secrets.GITHUB_TOKEN }} +# - name: Dogfooding codecov-cli +# if: ${{ !cancelled() && github.ref && contains(github.ref, 'pull') }} +# run: | +# codecovcli process-test-results --dir test_results --github-token ${{ secrets.GITHUB_TOKEN }}