Add workflow as option, grep the logs to check if it flaky test

This commit is contained in:
shubhamrasal 2023-03-03 06:13:17 +00:00
parent a92f429f86
commit c3d800fa41

View File

@ -1,9 +1,24 @@
#!/bin/bash #!/bin/bash
# This script is used to retry failed workflows in github actions.
# It uses gh cli to fetch the failed workflows and then rerun them.
# It also checks the logs of the failed workflows to see if it is a flaky test.
# If it is a flaky test, it will rerun the failed jobs in the workflow.
# eg:
# ./gh_retry.sh -h to see the help.
# ./gh_retry.sh will run the script with default values.
# You can also pass the following arguments:
# ./gh_retry.sh -b master -l 30 -t "30 mins ago" -w "Build Test"
#Initialize variables to default values. #Initialize variables to default values.
BRANCH=$(git symbolic-ref --short HEAD) BRANCH=$(git symbolic-ref --short HEAD)
LIMIT=300 LIMIT=30
BEFORE="30 mins ago" BEFORE="30 mins ago"
WORKFLOW="Build Test"
# You can add multiple patterns seperated by |
GREP_ERROR_PATTERN='Test "http/interactsh.yaml" failed'
#Set fonts for Help. #Set fonts for Help.
NORM=`tput sgr0` NORM=`tput sgr0`
@ -19,21 +34,25 @@ HELP()
echo "options:" echo "options:"
echo "${REV}-b${NORM} Branch to check failed workflows/jobs. Default is ${BOLD}$BRANCH${NORM}." echo "${REV}-b${NORM} Branch to check failed workflows/jobs. Default is ${BOLD}$BRANCH${NORM}."
echo "${REV}-l${NORM} Maximum number of runs to fetch. Default is ${BOLD}$LIMIT${NORM}." echo "${REV}-l${NORM} Maximum number of runs to fetch. Default is ${BOLD}$LIMIT${NORM}."
echo "${REV}-t${NORM} Time to filter the failed jobs . Default is ${BOLD}$BEFORE${NORM}." echo "${REV}-t${NORM} Time to filter the failed jobs. Default is ${BOLD}$BEFORE${NORM}."
echo "${REV}-w${NORM} Workflow to filter the failed jobs. Default is ${BOLD}$WORKFLOW${NORM}."
echo echo
} }
while getopts :b:l:t:h FLAG; do while getopts :b:l:t:w:h FLAG; do
case $FLAG in case $FLAG in
b) #set option "b" b)
BRANCH=$OPTARG BRANCH=$OPTARG
;; ;;
l) #set option "c" l)
LIMIT=$OPTARG LIMIT=$OPTARG
;; ;;
t) #set option "d" t)
BEFORE=$OPTARG BEFORE=$OPTARG
;; ;;
w)
WORKFLOW=$OPTARG
;;
h) #show help h) #show help
HELP HELP
exit 0 exit 0
@ -47,27 +66,40 @@ while getopts :b:l:t:h FLAG; do
done done
shift $((OPTIND-1)) shift $((OPTIND-1))
echo "Checking failed workflows for branch $BRANCH before $BEFORE" function print_bold() {
echo "${BOLD}$1${NORM}"
}
date=`date +%Y-%m-%d'T'%H:%M'Z' -d "$BEFORE"` function retry_failed_jobs() {
print_bold "Checking failed workflows for branch $BRANCH before $BEFORE"
workflowIds=$(gh run list --limit "$LIMIT" --json headBranch,status,name,conclusion,databaseId,updatedAt | jq -c '.[] | date=`date +%Y-%m-%d'T'%H:%M'Z' -d "$BEFORE"`
select ( .headBranch==$branch ) |
select ( .name | contains("Build Test") ) |
select ( .conclusion=="failure" ) |
select ( .updatedAt > $date) ' --arg date "$date" --arg branch "$BRANCH" | jq .databaseId)
# convert line seperated by space to array workflowIds=$(gh run list --limit "$LIMIT" --json headBranch,status,name,conclusion,databaseId,updatedAt | jq -c '.[] |
eval "arr=($workflowIds)" select ( .headBranch==$branch ) |
select ( .name | contains($workflow) ) |
select ( .conclusion=="failure" ) |
select ( .updatedAt > $date) ' --arg date "$date" --arg branch "$BRANCH" --arg workflow "$WORKFLOW" | jq .databaseId)
if [[ !${arr[@]} ]] # convert line seperated by space to array
then eval "arr=($workflowIds)"
echo "Could not find any failed workflows in the last $before"
fi
for s in "${arr[@]}"; do if [[ -z $arr ]]
echo "Retrying worklflow failed jobs $s" then
print_bold "Could not find any failed workflows in the last $BEFORE"
exit 0
fi
for s in "${arr[@]}"; do
print_bold "Checking logs of failed workflow $s to see if it is a flaky test"
gh run view "$s" --log-failed | grep -E "$GREP_ERROR_PATTERN" > /dev/null
if [ $? == 0 ] ; then
print_bold "Retrying failed jobs $s"
gh run rerun "$s" --failed gh run rerun "$s" --failed
sleep 10s sleep 10s
gh run view "$s" gh run view "$s"
done fi
done
}
retry_failed_jobs