[pw-ci] [RFC v3 1/2] github: support respositories with multiple workflows

Michael Santana msantana at redhat.com
Mon Aug 2 18:52:45 UTC 2021


On Thu, Jul 29, 2021 at 5:36 PM Salvatore Daniele <sdaniele at redhat.com> wrote:
>
> The existing github monitor code only supports a single workflow job.
> This worked as an initial implementation, but some projects are
> configuring multiple workflows and the monitor is unable to cope with
> this.
>
> To resolve this, we pull the workflow configurations and then ensure
> that all workflows are completed, and emit results once all have
> finished.  Each workflow run can now emit two 'pw|...' strings.
>
> As part of this change, we add a new field 'test_name to the 'pw|...'
> string to support passing the workflow name around.  CI systems like
> Travis won't report anything here so test_name will be blank for those
> systems.
>
> Signed-off-by: Salvatore Daniele <sdaniele at redhat.com>
> ---
>  ci_mon             | 12 +++++++++---
>  github_get_logs.sh | 11 ++++++++---
>  github_mon         | 37 ++++++++++++++++++++++++++-----------
>  3 files changed, 43 insertions(+), 17 deletions(-)
>
> diff --git a/ci_mon b/ci_mon
> index a6e4222..e2cdaf2 100755
> --- a/ci_mon
> +++ b/ci_mon
> @@ -149,7 +149,7 @@ for I in travis github dummy; do
>      fi
>
>      ./${I}_mon $pw_instance ${!token} "$pw_project" | grep "^pw|" | while IFS="|" \
> -        read -r PW pw_chk_instance BUILD series_id SHA shasum result build_url series_name repo_name; do
> +        read -r PW pw_chk_instance BUILD series_id SHA shasum result build_url series_name repo_name test_name; do
>
>          SERIES_LINE=$(./series_get $pw_instance $series_id)
>          sid=$(echo $SERIES_LINE | cut -d\| -f1)
> @@ -200,7 +200,13 @@ for I in travis github dummy; do
>          echo "References: $message_id" >> report.eml
>          echo "" >> report.eml
>
> -        echo "Test-Label: ${I}-robot" >> report.eml
> +        if [ "$test_name" != "" ]; then
> +            TEST_LABEL="${I}-robot: ${test_name}"
> +        else
> +            TEST_LABEL="${I}-robot"
> +        fi
> +
> +        echo "Test-Label: $TEST_LABEL" >> report.eml
>          echo "Test-Status: $RESULT" >> report.eml
>          echo "$patch_url" >> report.eml
>          echo "" >> report.eml
> @@ -208,7 +214,7 @@ for I in travis github dummy; do
>          echo "Build URL: $build_url" >> report.eml
>
>          if [ -f ./${I}_get_logs.sh ]; then
> -            ./${I}_get_logs.sh "$repo_name" "$series_id" "$shasum" "${!token}" >> report.eml
> +            ./${I}_get_logs.sh "$repo_name" "$series_id" "$shasum" "${!token}" "$test_name" >> report.eml
>          fi
>
>          if [ "$result" != "passed" ]; then
> diff --git a/github_get_logs.sh b/github_get_logs.sh
> index 60a763d..2f6cc9c 100755
> --- a/github_get_logs.sh
> +++ b/github_get_logs.sh
> @@ -41,8 +41,13 @@ if [ "X" = "X$github_token" ]; then
>      shift
>  fi
>
> -if [ "X" = "X$repo_name" -o "X" = "X$series_id" -o "X" = "X$sha" -o "X" = "X$github_token" ]; then
> -    echo "repo_name, series_id, sha or token were not passed in as an arugment. Exiting" 1>&2
> +if [ "X" = "X$test_name" ]; then
> +    test_name="$1"
> +    shift
> +fi
> +
> +if [ "X" = "X$repo_name" -o "X" = "X$series_id" -o "X" = "X$sha" -o "X" = "X$github_token" -o "X" = "X$test_name" ]; then
> +    echo "repo_name, series_id, sha, token, or test_name were not passed in as an arugment. Exiting" 1>&2
>      exit 1
>  fi
>
> @@ -61,7 +66,7 @@ GITHUB_API="https://api.github.com"
>  print_errored_logs_for_commit () {
>
>      # Get run metadata
> -    select="select(.head_branch==\"series_${series_id}\") | select(.head_sha==\"${sha}\")"
> +    select="select(.head_branch==\"series_${series_id}\") | select(.head_sha==\"${sha}\") | select(.name==\"${test_name}\")"
>      headers="{id, logs_url}"
>      run_meta="$(echo "$runs" | jq ".workflow_runs[] | $select | $headers")"
>
> diff --git a/github_mon b/github_mon
> index bc49f46..6ee604a 100755
> --- a/github_mon
> +++ b/github_mon
> @@ -58,28 +58,43 @@ make_result_for_series () {
>      sha="$5"
>      repo_name="$6"
>
> -    # Get run metadata
> +    # Get the name of each respective workflow
> +    workflows="$(echo "$runs" | jq -r ".workflow_runs[].name" | sort -u)"
> +
> +    # Check that workflows have all completed
>      select="select(.head_branch==\"series_${series_id}\") | select(.head_sha==\"${sha}\")"
>      headers="{status, conclusion, html_url}"
>      run_meta="$(echo "$runs" | jq ".workflow_runs[] | $select | $headers")"
>
> -    if [ "$(echo "$run_meta" | jq -r ".status")" != "completed" ]; then
> +    if [ "$(echo "$run_meta" | jq -r ".status" | sort -u)" != "completed" ]; then
Can you explain the reason for sort -u ? Does it have something to do
with multiple workflows? Does it make sure all workflows are
completed?
>          echo "patch_id=$patch_id belonging to series_id=$series_id not completed. Skipping" 1>&2
>          return 2
>      fi
>
> -    # success/failure?
> -    result="$(echo "$run_meta" | jq -r ".conclusion")"
> -    build_url="$(echo "$run_meta" | jq -r ".html_url")"
> +    # Scan relevant data for each workflow
> +    echo "$workflows" | while IFS='\n' read -r WORKFLOWNAME; do
>
> -    if [ "$result" == "success" ]; then
> -        result="passed"
> -    else
> -        result="failed"
> -    fi
> +        # Get run metadata
> +        select="select(.head_branch==\"series_${series_id}\") | select(.head_sha==\"${sha}\") | select(.name==\"${WORKFLOWNAME}\")"
> +        headers="{status, conclusion, html_url}"
> +        run_meta="$(echo "$runs" | jq ".workflow_runs[] | $select | $headers")"
> +
> +        # success/failure?
> +        result="$(echo "$run_meta" | jq -r ".conclusion")"
> +        build_url="$(echo "$run_meta" | jq -r ".html_url")"
> +
> +        if [ "$result" == "success" ]; then
> +            result="passed"
> +        else
> +            result="failed"
> +        fi
> +
> +        test_name=$WORKFLOWNAME
> +
> +        echo "pw|$pw_instance|build|$series_id|SHA|$sha|$result|$build_url|$patch_name|$repo_name|$test_name"
> +    done
>
>      set_synced_patch "$patch_id" "$pw_instance"
> -    echo "pw|$pw_instance|build|$series_id|SHA|$sha|$result|$build_url|$patch_name|$repo_name"
>  }
>
>  prev_series=""
> --
> 2.31.1
>
> _______________________________________________
> Pwci mailing list
> Pwci at lists.esnacc.org
> http://mail.esnacc.org/mailman/listinfo/pwci
>



More information about the Pwci mailing list