[pw-ci] [PATCH v4 2/2] obs_mon: Add OpenSuse Build Service support
Michael Santana
msantana at redhat.com
Tue Oct 5 20:11:31 UTC 2021
On Tue, Oct 5, 2021 at 4:08 PM Michael Santana <msantana at redhat.com> wrote:
>
> Add two scripts, one to get build status from obs and the other to get
> the build logs from obs. The script to get the logs from obs will also
> delete the build after retrieving the logs
>
> get_logs code is generic based on architecture and repository
>
> Signed-off-by: Michael Santana <msantana at redhat.com>
diff from v3 to v4:
added obs to ci_mon
obs_get_logs now looks through the xml file for repositories and
architectures rather than those being hardcoded
> ---
> ci_mon | 2 +-
> obs_get_logs.sh | 127 ++++++++++++++++++++++++++++++++++++++++++++++++
> obs_mon | 115 +++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 243 insertions(+), 1 deletion(-)
> create mode 100755 obs_get_logs.sh
> create mode 100755 obs_mon
>
> diff --git a/ci_mon b/ci_mon
> index 40b6432..adc5d81 100755
> --- a/ci_mon
> +++ b/ci_mon
> @@ -139,7 +139,7 @@ if [ "X" = "X$pw_instance" ]; then
> exit 1
> fi
>
> -for I in travis github dummy; do
> +for I in travis github obs dummy; do
>
> token=${I}_token
> disable=disable_${I}
> diff --git a/obs_get_logs.sh b/obs_get_logs.sh
> new file mode 100755
> index 0000000..29707b3
> --- /dev/null
> +++ b/obs_get_logs.sh
> @@ -0,0 +1,127 @@
> +#!/bin/bash
> +# SPDX-Identifier: gpl-2.0-or-later
> +# Copyright (C) 2021, Red Hat, Inc.
> +#
> +# Prints out openSUSE Build Service build logs for failed jobs, when
> +# provided wtih project (e.g. "home:rhobsrobot:dpdk"), series_id.
> +# Although shasum and token are not used, they must be passed to be
> +# consistent with other *_mon scripts. Pass in "NULL" for those parameters.
> +#
> +# Licensed under the terms of the GNU General Public License as published
> +# by the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version. You may obtain a copy of the
> +# license at
> +#
> +# https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
> +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
> +# License for the specific language governing permissions and limitations
> +# under the License.
> +
> +
> +[ -f "${HOME}/.obs_ci_mon_rc" ] && source "${HOME}/.obs_ci_mon_rc"
> +
> +# Also known as repo_name in github_actions
> +if [ "X" = "X$project" ]; then
> + project="$1"
> + shift
> +fi
> +
> +if [ "X" = "X$series_id" ]; then
> + series_id="$1"
> + shift
> +fi
> +
> +if [ "X" = "X$sha" ]; then
> + sha="$1"
> + shift
> +fi
> +
> +if [ "X" = "X$obs_token" ]; then
> + obs_token="$1"
> + shift
> +fi
> +
> +if [ "X" = "X$project" -o "X" = "X$series_id" -o "X" = "X$sha" -o "X" = "X$obs_token" ]; then
> + echo "project, series_id, sha or token were not passed in as an arugment. Exiting" 1>&2
> + exit 1
> +fi
> +
> +OSCBIN=osc
> +res_file=/tmp/obs_build_results.xml
> +path="//status[@package=\"$series_id\"]/@code"
> +$OSCBIN -c ~/.dpdkoscconf api -X GET /build/$project/_result > $res_file
> +
> +repository="$(xmllint --xpath "/resultlist//@repository" $res_file | sort | uniq | sed 's/ *repository="\(.*\)".*/\1/' | tr '\n' ' ')"
> +arch="$(xmllint --xpath "/resultlist//@arch" $res_file | sort | uniq | sed 's/ *arch="\(.*\)".*/\1/' | tr '\n' ' ')"
> +
> +# Make sure there is something to process
> +series_results="$(xmllint --xpath "$path" $res_file)"
> +if [ $? -ne 0 ]; then
> + echo -n "Error parsing api response. " 1>&2
> + echo "Perhaps series=$series_id is not in the api result below:" 1>&2
> + cat $res_file 1>&2
> + echo "End of api response. Skipping series $series_id" 1>&2
> + exit 1
> +fi
> +
> +# Print summary of failed jobs
> +echo "---------------------------------Summary of failed jobs-----------------------------"
> +for arch_iter in $arch; do
> + for repo_iter in $repository; do
> + p1="//result[@repository=\"${repo_iter}\" and @arch=\"${arch_iter}\"]"
> + p2="status[@package=\"$series_id\"]/@code"
> +
> + # Get result (i.e. success, failed)
> + code="$(xmllint --xpath "$p1/$p2" $res_file 2>/dev/null)"
> +
> + if echo "$code" | grep -iq failed; then
> + echo failure on repo: $repo_iter --- arch: $arch_iter
> + fi
> + done
> +done
> +echo "-------------------------------End summary of failed jobs---------------------------"
> +
> +
> +# Print logs of failed jobs
> +echo -ne "\n\n\n\n"
> +echo "---------------------------------------BEGIN LOGS-----------------------------------"
> +spacing=0
> +for arch_iter in $arch; do
> + for repo_iter in $repository; do
> + p1="//result[@repository=\"${repo_iter}\" and @arch=\"${arch_iter}\"]"
> + p2="status[@package=\"$series_id\"]/@code"
> + p3="/build/$project/${repo_iter}/${arch_iter}/$series_id/_log"
> +
> + # Get result (i.e. success, failed)
> + code="$(xmllint --xpath "$p1/$p2" $res_file 2>/dev/null)"
> +
> + if echo "$code" | grep -iq failed; then
> + if [ ! "$spacing" -eq "0" ]
> + then
> + echo -ne "\n\n\n\n"
> + fi
> +
> + echo "################################################################################"
> + echo "#### [Begin job log] repo: \"$repo_iter\", arch: \"$arch_iter\""
> + echo "################################################################################"
> +
> + echo failure on $arch_iter and $repo_iter
> +
> + # Get failed logs and print them out
> + $OSCBIN -c ~/.dpdkoscconf api -X GET "$p3" | tail -n 50
> +
> + echo "################################################################################"
> + echo "#### [End job log] repo: \"$repo_iter\", arch: \"$arch_iter\""
> + echo "################################################################################"
> +
> + spacing=1
> + fi
> + done
> +done
> +echo "------------------------------------END LOGS------------------------------------"
> +
> +# Delete series after getting the logs
> +$OSCBIN -c ~/.dpdkoscconf api -X DELETE /source/$project/$series_id > /dev/null
> diff --git a/obs_mon b/obs_mon
> new file mode 100755
> index 0000000..294946f
> --- /dev/null
> +++ b/obs_mon
> @@ -0,0 +1,115 @@
> +#!/bin/bash
> +# SPDX-Identifier: gpl-2.0-or-later
> +# Copyright (C) 2021, Red Hat, Inc.
> +#
> +# Monitors a openSUSE Build Service build history for builds in a series.
> +# Records the builds in the series database (and emits them on the
> +# stdout line for processing)
> +#
> +# Licensed under the terms of the GNU General Public License as published
> +# by the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version. You may obtain a copy of the
> +# license at
> +#
> +# https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
> +#
> +# Unless required by applicable law or agreed to in writing, software
> +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
> +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
> +# License for the specific language governing permissions and limitations
> +# under the License.
> +
> +[ -f "$(dirname $0)/series_db_lib.sh" ] && source "$(dirname $0)/series_db_lib.sh"
> +[ -f "${HOME}/.obs_ci_mon_rc" ] && source "${HOME}/.obs_ci_mon_rc"
> +
> +if [ "X" = "X$pw_instance" ]; then
> + pw_instance="$1"
> + shift
> +fi
> +
> +if [ "X" = "X$obs_token" ]; then
> + obs_token="$1"
> + shift
> +fi
> +
> +if [ "X" = "X$pw_project" -a "X$1" != "X" ]; then
> + pw_project="$1"
> + shift
> +fi
> +
> +if [ "X" = "X$pw_instance" -o "X" = "X$obs_token" ]; then
> + echo "pw_instance or obs_token were not set." 1>&2
> + echo -n "These can be passed as an argument or set in " 1>&2
> + echo "${HOME}/.obs_ci_mon_rc. Exiting" 1>&2
> + exit 1
> +fi
> +
> +OSCBIN=osc
> +ci_instance=obs_sync
> +res_file=/tmp/obs_build_results.xml
> +
> +make_result_for_series () {
> +
> + series_id="$1"
> + patch_id="$2"
> + patch_url="$3"
> + patch_name="$4"
> + sha="$5"
> + repo_name="$6"
> + result=err
> +
> + if echo "$series_results" | grep -iq building || \
> + echo "$series_results" | grep -iq scheduled; then
> + echo -n "patch_id=$patch_id belonging to series_id=$series_id not" 1>&2
> + echo -e " completed. Skipping\n" 1>&2
> + echo "$series_results" 1>&2
> + return 1
> + fi
> +
> + echo "Build for series $series_id finished." 1>&2
> +
> + # Check if successful, overwrite success if failure
> + if echo "$series_results" | grep -iq succeeded; then
> + result=success
> + fi
> + if echo "$series_results" | grep -iq failed; then
> + result=failure
> + fi
> + if [ "$result" == "err" ]; then
> + echo "Error parsing result from series_results below:" 1>&2
> + echo "$series_results" 1>&2
> + return 2
> + fi
> +
> + set_synced_patch "$patch_id" "$pw_instance" "$ci_instance"
> + echo "pw|$pw_instance|build|$series_id|SHA|$sha|$result|$build_url|$patch_name|$repo_name|OBS"
> +}
> +
> +prev_series=""
> +
> +get_unsynced_series "$pw_instance" "$ci_instance" | \
> +while IFS="|" read -r series_id patch_id patch_url patch_name sha patchwork_instance patchwork_project repo_name gap_sync; do
> +
> + if [ "X$pw_project" != "X" -a "X$pw_project" != "X$patchwork_project" ]; then
> + continue
> + fi
> +
> + if [ "X$series_id" != "X$prev_series" ]; then
> + prev_series="$series_id"
> + path="//status[@package=\"series_$series_id\"]/@code"
> + $OSCBIN -c ~/.dpdkoscconf api -X GET /build/$repo_name/_result > $res_file
> + build_url="https://build.opensuse.org/repo_name/show/$repo_name"
> +
> + # Get OBS runs
> + series_results="$(xmllint --xpath "$path" $res_file)"
> + if [ $? -ne 0 ]; then
> + echo -n "Error parsing api response. " 1>&2
> + echo "Perhaps series=$series_id is not in the api result below:" 1>&2
> + cat $res_file 1>&2
> + echo "End of api response. Skipping series $series_id"
> + continue
> + fi
> + fi
> +
> + make_result_for_series "$series_id" "$patch_id" "$patch_url" "$patch_name" "$sha" "$repo_name"
> +done
> --
> 2.31.1
>
More information about the Pwci
mailing list