From fa816bfa22b06a543ce1e255a213b7edc2e84803 Mon Sep 17 00:00:00 2001 From: XiaochaoGONG Date: Thu, 17 Oct 2019 21:42:33 +0800 Subject: [PATCH] Misc/gitlab ci --- .gitlab-ci.yml | 55 ++++++++-- tools/ci/build_examples.sh | 38 +++---- tools/ci/build_examples_cmake.sh | 173 +++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 32 deletions(-) create mode 100755 tools/ci/build_examples_cmake.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8149da2..2eb1673 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,29 +66,61 @@ before_script: artifacts: when: always paths: - - build_examples/*/*/build/*.bin - - build_examples/*/*/build/*.elf - - build_examples/*/*/build/*.map - - build_examples/*/*/build/download.config - - build_examples/*/*/build/bootloader/*.bin + - build_examples_make/*/*/build/*.bin + - build_examples_make/*/*/build/*.elf + - build_examples_make/*/*/build/*.map + - build_examples_make/*/*/build/bootloader/*.bin - ${LOG_PATH} - ${IDF_PATH}/tools/ci expire_in: 1 week variables: - IDF_CI_BUILD: "1" LOG_PATH: "$CI_PROJECT_DIR/log_examples_make" script: # it's not possible to build 100% out-of-tree and have the "artifacts" # mechanism work, but this is the next best thing - - rm -rf build_examples - - mkdir build_examples - - cd build_examples + - rm -rf build_examples_make + - mkdir build_examples_make + - cd build_examples_make - mkdir -p ${LOG_PATH} - - ${CI_PROJECT_DIR}/tools/ci/build_examples.sh "${CI_PROJECT_DIR}/examples" "${CI_JOB_NAME}" + - ${CI_PROJECT_DIR}/tools/ci/build_examples.sh + +.build_examples_cmake_template: &build_examples_cmake_template + <<: *build_template + retry: 1 + artifacts: + when: always + paths: + - build_examples_cmake/*/*/build/*.bin + - build_examples_cmake/*/*/build/*.elf + - build_examples_cmake/*/*/build/*.map + - build_examples_cmake/*/*/*/build/flasher_args.json + - build_examples_cmake/*/*/build/bootloader/*.bin + - build_examples_cmake/*/*/build/partition_table/*.bin + - ${LOG_PATH} + - ${IDF_PATH}/tools/ci + expire_in: 1 week + variables: + LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake" + script: + # it's not possible to build 100% out-of-tree and have the "artifacts" + # mechanism work, but this is the next best thing + - rm -rf build_examples_cmake + - mkdir build_examples_cmake + - cd build_examples_cmake + - mkdir -p ${LOG_PATH} + - ${CI_PROJECT_DIR}/tools/ci/build_examples_cmake.sh build_examples_make_00: <<: *build_examples_make_template +build_examples_make_01: + <<: *build_examples_make_template + +build_examples_cmake_00: + <<: *build_examples_cmake_template + +build_examples_cmake_01: + <<: *build_examples_cmake_template push_to_github: stage: deploy @@ -101,6 +133,9 @@ push_to_github: when: on_success dependencies: - build_examples_make_00 + - build_examples_make_01 + - build_examples_cmake_00 + - build_examples_cmake_01 before_script: - echo "skip default before_script" script: diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index b24eed8..6c8ebec 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -48,9 +48,6 @@ die() { echo "build_examples running in ${PWD}" -# only 0 or 1 arguments -[ $# -eq 2 ] || die "Have to run as $(basename $0) [] []" - export BATCH_BUILD=1 export V=0 # only build verbose if there's an error @@ -61,31 +58,27 @@ FAILED_EXAMPLES="" RESULT_ISSUES=22 # magic number result code for issues found LOG_SUSPECTED=${LOG_PATH}/common_log.txt touch ${LOG_SUSPECTED} +SDKCONFIG_DEFAULTS_CI=sdkconfig.ci -if [ $# -eq 0 ] +EXAMPLE_PATHS=$( find ${CI_PROJECT_DIR}/examples/ -type f -name Makefile | grep -v "/build_system/cmake/" | sort ) + +if [ -z "${CI_NODE_TOTAL:-}" ] then START_NUM=0 END_NUM=999 else - JOB_PATH=$1 - JOB_NAME=$2 - - # parse text prefix at the beginning of string 'some_your_text_NUM' - # (will be 'some_your_text' without last '_') - JOB_PATTERN=$( echo ${JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' ) + JOB_PATTERN=$( echo ${CI_JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' ) [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad" - - # parse number 'NUM' at the end of string 'some_your_text_NUM' - # NOTE: Getting rid of the leading zero to get the decimal - JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' ) + + JOB_NUM=$( echo ${CI_JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' ) [ -z ${JOB_NUM} ] && die "JOB_NUM is bad" - + # count number of the jobs - NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${CI_PROJECT_DIR}/.gitlab-ci.yml" ) + NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${CI_PROJECT_DIR}/.gitlab-ci.yml" ) [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad" # count number of examples - NUM_OF_EXAMPLES=$( find ${JOB_PATH} -type f -name Makefile | wc -l ) + NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l ) [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad" # separate intervals @@ -124,7 +117,7 @@ build_example () { make clean >>${BUILDLOG} 2>&1 && - make defconfig >>${BUILDLOG} 2>&1 && + make defconfig >>${BUILDLOG} 2>&1 && make all >>${BUILDLOG} 2>&1 && cat ${BUILDLOG} @@ -135,17 +128,18 @@ build_example () { EXAMPLE_NUM=0 -find ${JOB_PATH} -type f -name Makefile | sort | \ -while read FN +echo "Current job will build example ${START_NUM} - ${END_NUM}" + +for EXAMPLE_PATH in ${EXAMPLE_PATHS} do if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]] then EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) continue fi - echo ">>> example [ ${EXAMPLE_NUM} ] - $FN" + echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH" - build_example "${EXAMPLE_NUM}" "${FN}" + build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}" EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) done diff --git a/tools/ci/build_examples_cmake.sh b/tools/ci/build_examples_cmake.sh new file mode 100755 index 0000000..b6d9c01 --- /dev/null +++ b/tools/ci/build_examples_cmake.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# +# Build all examples from the examples directory, out of tree to +# ensure they can run when copied to a new directory. +# +# Runs as part of CI process. +# +# Assumes PWD is an out-of-tree build directory, and will copy examples +# to individual subdirectories, one by one. +# +# +# Without arguments it just builds all examples +# +# With one argument it builds part of the examples. This is a useful for +# parallel execution in CI. +# must look like this: +# _ +# It scans .gitlab-ci.yaml to count number of jobs which have name "_" +# It scans the filesystem to count all examples +# Based on this, it decides to run qa set of examples. +# + +# ----------------------------------------------------------------------------- +# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d). + +if [[ ! -z ${DEBUG_SHELL} ]] +then + set -x # Activate the expand mode if DEBUG is anything but empty. +fi + +set -o errexit # Exit if command failed. +set -o pipefail # Exit if pipe failed. +set -o nounset # Exit if variable not set. + +# Remove the initial space and instead use '\n'. +IFS=$'\n\t' + +export PATH="$IDF_PATH/tools:$PATH" # for idf.py + +# ----------------------------------------------------------------------------- + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +[ -z ${IDF_PATH} ] && die "IDF_PATH is not set" +[ -z ${LOG_PATH} ] && die "LOG_PATH is not set" +[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} + +echo "build_examples running in ${PWD}" + +export BATCH_BUILD=1 +export V=0 # only build verbose if there's an error + +shopt -s lastpipe # Workaround for Bash to use variables in loops (http://mywiki.wooledge.org/BashFAQ/024) + +RESULT=0 +FAILED_EXAMPLES="" +RESULT_ISSUES=22 # magic number result code for issues found +LOG_SUSPECTED=${LOG_PATH}/common_log.txt +touch ${LOG_SUSPECTED} +SDKCONFIG_DEFAULTS_CI=sdkconfig.ci + +EXAMPLE_PATHS=$( find ${CI_PROJECT_DIR}/examples/ -type f -name CMakeLists.txt | grep -v "/components/" | grep -v "/main/" | sort ) + +if [ -z "${CI_NODE_TOTAL:-}" ] +then + START_NUM=0 + END_NUM=999 +else + JOB_PATTERN=$( echo ${CI_JOB_NAME} | sed -n -r 's/^(.*)_[0-9]+$/\1/p' ) + [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad" + + JOB_NUM=$( echo ${CI_JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' ) + [ -z ${JOB_NUM} ] && die "JOB_NUM is bad" + + # count number of the jobs + NUM_OF_JOBS=$( grep -c -E "^${JOB_PATTERN}_[0-9]+:$" "${CI_PROJECT_DIR}/.gitlab-ci.yml" ) + [ -z ${NUM_OF_JOBS} ] && die "NUM_OF_JOBS is bad" + + # count number of examples + NUM_OF_EXAMPLES=$( echo "${EXAMPLE_PATHS}" | wc -l ) + [ -z ${NUM_OF_EXAMPLES} ] && die "NUM_OF_EXAMPLES is bad" + + # separate intervals + #57 / 5 == 12 + NUM_OF_EX_PER_JOB=$(( (${NUM_OF_EXAMPLES} + ${NUM_OF_JOBS} - 1) / ${NUM_OF_JOBS} )) + [ -z ${NUM_OF_EX_PER_JOB} ] && die "NUM_OF_EX_PER_JOB is bad" + + # ex.: [0; 12); [12; 24); [24; 36); [36; 48); [48; 60) + START_NUM=$(( ${JOB_NUM} * ${NUM_OF_EX_PER_JOB} )) + [ -z ${START_NUM} ] && die "START_NUM is bad" + + END_NUM=$(( (${JOB_NUM} + 1) * ${NUM_OF_EX_PER_JOB} )) + [ -z ${END_NUM} ] && die "END_NUM is bad" +fi + +build_example () { + local ID=$1 + shift + local CMAKELISTS=$1 + shift + + local EXAMPLE_DIR=$(dirname "${CMAKELISTS}") + local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}") + + echo "Building ${EXAMPLE_NAME} as ${ID}..." + mkdir -p "${ID}" + cp -r "${EXAMPLE_DIR}" "${ID}" + pushd "${ID}/${EXAMPLE_NAME}" + # be stricter in the CI build than the default IDF settings + export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" + export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} + + # build non-verbose first + local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt + touch ${BUILDLOG} + + + idf.py fullclean >>${BUILDLOG} 2>&1 && + idf.py build >>${BUILDLOG} 2>&1 && + + cat ${BUILDLOG} + popd + + grep -i "error\|warning" "${BUILDLOG}" 2>&1 | grep -v "error.c.obj" >> "${LOG_SUSPECTED}" || : +} + +EXAMPLE_NUM=0 + +echo "Current job will build example ${START_NUM} - ${END_NUM}" + +for EXAMPLE_PATH in ${EXAMPLE_PATHS} +do + if [[ $EXAMPLE_NUM -lt $START_NUM || $EXAMPLE_NUM -ge $END_NUM ]] + then + EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) + continue + fi + echo ">>> example [ ${EXAMPLE_NUM} ] - $EXAMPLE_PATH" + + build_example "${EXAMPLE_NUM}" "${EXAMPLE_PATH}" + + EXAMPLE_NUM=$(( $EXAMPLE_NUM + 1 )) +done + +# show warnings +echo -e "\nFound issues:" + +# Ignore the next messages: +# "error.o" or "-Werror" in compiler's command line +# "reassigning to symbol" or "changes choice state" in sdkconfig +# 'Compiler and toochain versions is not supported' from crosstool_version_check.cmake +IGNORE_WARNS="\ +library/error\.o\ +\|\ -Werror\ +\|error\.d\ +\|reassigning to symbol\ +\|changes choice state\ +\|crosstool_version_check\.cmake\ +" + +sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \ + && RESULT=$RESULT_ISSUES \ + || echo -e "\tNone" + +[ -z ${FAILED_EXAMPLES} ] || echo -e "\nThere are errors in the next examples: $FAILED_EXAMPLES" +[ $RESULT -eq 0 ] || echo -e "\nFix all warnings and errors above to pass the test!" + +echo -e "\nReturn code = $RESULT" + +exit $RESULT