diff --git a/.ci/Jenkinsfile-compile b/.ci/Jenkinsfile-compile new file mode 100644 index 0000000000..f76f6340e3 --- /dev/null +++ b/.ci/Jenkinsfile-compile @@ -0,0 +1,242 @@ +#!/usr/bin/env groovy + +pipeline { + agent none + stages { + + stage('Build') { + steps { + script { + def build_nodes = [:] + def docker_images = [ + arch: "px4io/px4-dev-base-archlinux:2018-07-19", + armhf: "px4io/px4-dev-armhf:2018-07-19", + base: "px4io/px4-dev-base:2018-07-19", + clang: "px4io/px4-dev-clang:2018-07-19", + nuttx: "px4io/px4-dev-nuttx:2018-07-19", + ros: "px4io/px4-dev-ros:2018-07-19", + rpi: "px4io/px4-dev-raspi:2018-07-19", + snapdragon: "lorenzmeier/px4-dev-snapdragon:2017-12-29" + ] + + // special builds: + // fmu-v2_{default, lpe} and fmu-v3_{default, rtps} + // bloaty compare to last successful master build + build_nodes["px4fmu-v2"] = { + node { + stage("Build Test px4fmu-v2") { + docker.image(docker_images.nuttx).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { + stage("px4fmu-v2") { + try { + checkout(scm) + sh('export') + sh('make distclean') + sh('ccache -z') + sh('git fetch --tags') + sh('make nuttx_px4io-v2_default') + sh('make nuttx_px4io-v2_default bloaty_symbols') + sh('make nuttx_px4io-v2_default bloaty_compileunits') + sh('make nuttx_px4io-v2_default bloaty_compare_master') + sh('make nuttx_px4fmu-v2_default') + sh('make nuttx_px4fmu-v2_default bloaty_symbols') + sh('make nuttx_px4fmu-v2_default bloaty_compileunits') + sh('make nuttx_px4fmu-v2_default bloaty_inlines') + sh('make nuttx_px4fmu-v2_default bloaty_templates') + sh('make nuttx_px4fmu-v2_default bloaty_compare_master') + sh('make nuttx_px4fmu-v2_lpe') + sh('make nuttx_px4fmu-v2_test') + sh('make nuttx_px4fmu-v3_default') + sh('make nuttx_px4fmu-v3_default bloaty_compare_master') + sh('make nuttx_px4fmu-v3_rtps') + sh('make sizes') + sh('ccache -s') + archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true) + } + catch (exc) { + throw (exc) + } + finally { + sh('make distclean') + } + } + } + } + } + } + + // MAC OSX posix_sitl_default + build_nodes["posix_sitl_default (OSX)"] = { + node("mac") { + withEnv(["CCACHE_BASEDIR=${pwd()}"]) { + stage("sitl (OSX)") { + try { + checkout(scm) + sh('export') + sh('make distclean') + sh('ccache -z') + sh('make posix_sitl_default') + sh('ccache -s') + } + catch (exc) { + throw (exc) + } + finally { + sh('make distclean') + } + } + } + } + } + + // MAC OSX nuttx_px4fmu-v4pro_default + build_nodes["px4fmu-v4pro_default (OSX)"] = { + node("mac") { + withEnv(["CCACHE_BASEDIR=${pwd()}"]) { + stage("px4fmu-v4pro (OSX)") { + try { + checkout(scm) + sh('export') + sh('make distclean') + sh('ccache -z') + sh('make nuttx_px4fmu-v4pro_default') + sh('ccache -s') + } + catch (exc) { + throw (exc) + } + finally { + sh('make distclean') + } + } + } + } + } + + // docker builds: + def arch_builds = [ + target: ["posix_sitl_default"], + image: docker_images.arch, + archive: false + ] + + def armhf_builds = [ + target: ["posix_ocpoc_ubuntu"], + image: docker_images.armhf, + archive: false + ] + + def base_builds = [ + target: ["posix_sitl_rtps"], + image: docker_images.base, + archive: false + ] + + def nuttx_builds_archive = [ + target: ["px4fmu-v4_default", "px4fmu-v4pro_default", "px4fmu-v5_default", "aerofc-v1_default", + "aerocore2_default", "auav-x21_default", "crazyflie_default", "mindpx-v2_default", + "nxphlite-v3_default", "tap-v1_default", "omnibus-f4sd_default"], + image: docker_images.nuttx, + archive: true + ] + + def nuttx_builds_other = [ + target: ["px4-same70xplained-v1_default", "px4-stm32f4discovery_default", "px4cannode-v1_default", + "px4esc-v1_default", "px4nucleoF767ZI-v1_default", "s2740vc-v1_default"], + image: docker_images.nuttx, + archive: false + ] + + def rpi_builds = [ + target: ["posix_rpi_cross", "posix_bebop_default"], + image: docker_images.rpi, + archive: false + ] + + def snapdragon_builds = [ + target: ["qurt_eagle_default", "posix_eagle_default"], + image: docker_images.snapdragon, + archive: false + ] + + def docker_builds = [ + arch_builds, armhf_builds, base_builds, nuttx_builds_archive, nuttx_builds_other, + rpi_builds, snapdragon_builds + ] + + for (def build_type = 0; build_type < docker_builds.size(); build_type++) { + for (def build_target = 0; build_target < docker_builds[build_type].target.size(); build_target++) { + build_nodes.put(docker_builds[build_type].target[build_target], + createBuildNode(docker_builds[build_type].archive, docker_builds[build_type].image, docker_builds[build_type].target[build_target]) + ) + } + } + + parallel build_nodes + + } // script + } // steps + } // stage Build + + // TODO: actually upload artifacts to S3 + stage('S3 Upload') { + agent { + docker { image 'px4io/px4-dev-base:2018-07-19' } + } + options { + skipDefaultCheckout() + } + when { + anyOf { + branch 'master' + branch 'beta' + branch 'stable' + branch 'pr-jenkins' // for testing + } + } + steps { + sh 'echo "uploading to S3"' + } + } + + } // stages + environment { + CCACHE_DIR = '/tmp/ccache' + CI = true + } + options { + buildDiscarder(logRotator(numToKeepStr: '10', artifactDaysToKeepStr: '30')) + timeout(time: 60, unit: 'MINUTES') + } +} + +def createBuildNode(Boolean archive, String docker_image, String target) { + return { + node { + docker.withRegistry('https://registry.hub.docker.com', 'docker_hub_dagar') { + docker.image(docker_image).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { + stage(target) { + try { + sh('export') + checkout(scm) + sh('make distclean') + sh('git fetch --tags') + sh('ccache -z') + sh('make ' + target) + sh('ccache -s') + sh('make sizes') + if (archive) { + archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf, build/**/*.bin', fingerprint: true, onlyIfSuccessful: true) + } + } + catch (exc) { + throw (exc) + } + finally { + sh('make distclean') + } + } + } + } + } + } +} diff --git a/Jenkinsfile b/Jenkinsfile index 37cae9214f..8eae271d96 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,141 +2,6 @@ pipeline { agent none stages { - stage('Build') { - steps { - script { - def builds = [:] - - def docker_base = "px4io/px4-dev-base:2018-07-19" - def docker_nuttx = "px4io/px4-dev-nuttx:2018-07-19" - def docker_ros = "px4io/px4-dev-ros:2018-07-19" - def docker_rpi = "px4io/px4-dev-raspi:2018-07-19" - def docker_armhf = "px4io/px4-dev-armhf:2018-07-19" - def docker_arch = "px4io/px4-dev-base-archlinux:2018-07-19" - def docker_snapdragon = "lorenzmeier/px4-dev-snapdragon:2017-12-29" - def docker_clang = "px4io/px4-dev-clang:2018-07-19" - - // fmu-v2_{default, lpe} and fmu-v3_{default, rtps} - // bloaty compare to last successful master build - builds["px4fmu-v2"] = { - node { - stage("Build Test px4fmu-v2") { - docker.image(docker_nuttx).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("px4fmu-v2") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "git fetch --tags" - sh "make nuttx_px4io-v2_default" - sh "make nuttx_px4io-v2_default bloaty_symbols" - sh "make nuttx_px4io-v2_default bloaty_compileunits" - sh "make nuttx_px4io-v2_default bloaty_compare_master" - sh "make nuttx_px4fmu-v2_default" - sh "make nuttx_px4fmu-v2_default bloaty_symbols" - sh "make nuttx_px4fmu-v2_default bloaty_compileunits" - sh "make nuttx_px4fmu-v2_default bloaty_inlines" - sh "make nuttx_px4fmu-v2_default bloaty_templates" - sh "make nuttx_px4fmu-v2_default bloaty_compare_master" - sh "make nuttx_px4fmu-v2_lpe" - sh "make nuttx_px4fmu-v2_test" - sh "make nuttx_px4fmu-v3_default" - sh "make nuttx_px4fmu-v3_rtps" - sh "make sizes" - sh "ccache -s" - archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true) - sh "make distclean" - } - } - } - } - } - - // nuttx default targets that are archived and uploaded to s3 - for (def option in ["px4fmu-v4", "px4fmu-v4pro", "px4fmu-v5", "aerofc-v1", "aerocore2", "av-x-v1", "auav-x21", "crazyflie", "mindpx-v2", "nxphlite-v3", "tap-v1", "omnibus-f4sd"]) { - def node_name = "${option}" - builds[node_name] = createBuildNodeArchive(docker_nuttx, "${node_name}_default") - } - - // other nuttx default targets - for (def option in ["px4-same70xplained-v1", "px4-stm32f4discovery", "px4cannode-v1", "px4esc-v1", "px4nucleoF767ZI-v1", "s2740vc-v1"]) { - def node_name = "${option}" - builds[node_name] = createBuildNode(docker_nuttx, "${node_name}_default") - } - - builds["sitl_rtps"] = createBuildNode(docker_base, 'posix_sitl_rtps') - builds["sitl (GCC 7)"] = createBuildNode(docker_arch, 'posix_sitl_default') - - builds["rpi"] = createBuildNode(docker_rpi, 'posix_rpi_cross') - builds["bebop"] = createBuildNode(docker_rpi, 'posix_bebop_default') - - builds["ocpoc"] = createBuildNode(docker_armhf, 'posix_ocpoc_ubuntu') - - // snapdragon (eagle_default) - builds["eagle (linux)"] = createBuildNodeDockerLogin(docker_snapdragon, 'docker_hub_dagar', 'posix_eagle_default') - builds["eagle (qurt)"] = createBuildNodeDockerLogin(docker_snapdragon, 'docker_hub_dagar', 'qurt_eagle_default') - - // posix_sitl_default with package - builds["sitl"] = { - node { - stage("Build Test sitl") { - docker.image(docker_ros).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE') { - stage("sitl") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make posix_sitl_default" - sh "make posix_sitl_default sitl_gazebo" - sh "make posix_sitl_default package" - sh "ccache -s" - stash name: "px4_sitl_package", includes: "build/posix_sitl_default/*.bz2" - sh "make distclean" - } - } - } - } - } - - // MAC OS posix_sitl_default - builds["sitl (OSX)"] = { - node("mac") { - withEnv(["CCACHE_BASEDIR=${pwd()}"]) { - stage("sitl (OSX)") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make posix_sitl_default" - sh "ccache -s" - sh "make distclean" - } - } - } - } - - // MAC OS nuttx_px4fmu-v4pro_default - builds["px4fmu-v4pro (OSX)"] = { - node("mac") { - withEnv(["CCACHE_BASEDIR=${pwd()}"]) { - stage("px4fmu-v4pro (OSX)") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make nuttx_px4fmu-v4pro_default" - sh "ccache -s" - sh "make distclean" - } - } - } - } - - parallel builds - } // script - } // steps - } // stage Builds - stage('Test') { parallel { @@ -178,6 +43,7 @@ pipeline { branch 'master' branch 'beta' branch 'stable' + branch 'pr-jenkins' // for testing } } } @@ -225,6 +91,7 @@ pipeline { branch 'master' branch 'beta' branch 'stable' + branch 'pr-jenkins' // for testing } } } @@ -324,70 +191,6 @@ pipeline { } } - stage('ROS MC mission box (EKF2)') { - agent { - docker { - image 'px4io/px4-dev-ros:2018-07-19' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - options { - skipDefaultCheckout() - } - steps { - sh 'export' - sh 'rm -rf build; rm -rf px4-posix_sitl_default*; rm -rf .ros; rm -rf .gazebo' - unstash 'px4_sitl_package' - sh 'tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2' - sh 'px4-posix_sitl_default*/px4/test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=multirotor_box vehicle:=iris' - sh 'px4-posix_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py `find . -name *.ulg -print -quit`' - } - post { - always { - sh 'px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - archiveArtifacts '.ros/**/*.pdf' - archiveArtifacts '.ros/**/*.csv' - deleteDir() - } - failure { - sh 'ls -a' - archiveArtifacts '.ros/**/*.ulg' - archiveArtifacts '.ros/**/rosunit-*.xml' - archiveArtifacts '.ros/**/rostest-*.log' - } - } - } - - stage('ROS MC mission box (LPE)') { - agent { - docker { - image 'px4io/px4-dev-ros:2018-07-19' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - options { - skipDefaultCheckout() - } - steps { - sh 'export' - sh 'rm -rf build; rm -rf px4-posix_sitl_default*; rm -rf .ros; rm -rf .gazebo' - unstash 'px4_sitl_package' - sh 'tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2' - sh 'px4-posix_sitl_default*/px4/test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=multirotor_box vehicle:=iris est:=lpe' - } - post { - always { - sh 'px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - deleteDir() - } - failure { - sh 'ls -a' - archiveArtifacts '.ros/**/*.ulg' - archiveArtifacts '.ros/**/rosunit-*.xml' - archiveArtifacts '.ros/**/rostest-*.log' - } - } - } } } @@ -450,25 +253,6 @@ pipeline { } } - // TODO: actually upload artifacts to S3 - stage('S3 Upload') { - agent { - docker { image 'px4io/px4-dev-base:2018-07-19' } - } - options { - skipDefaultCheckout() - } - when { - anyOf { - branch 'master' - branch 'beta' - branch 'stable' - } - } - steps { - sh 'echo "uploading to S3"' - } - } } // stages environment { @@ -480,66 +264,3 @@ pipeline { timeout(time: 60, unit: 'MINUTES') } } - -def createBuildNode(String docker_repo, String target) { - return { - node { - docker.image(docker_repo).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage(target) { - sh('export') - checkout scm - sh('make distclean') - sh('git fetch --tags') - sh('ccache -z') - sh('make ' + target) - sh('ccache -s') - sh('make sizes') - sh('make distclean') - } - } - } - } -} - -def createBuildNodeArchive(String docker_repo, String target) { - return { - node { - docker.image(docker_repo).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage(target) { - sh('export') - checkout scm - sh('make distclean') - sh('git fetch --tags') - sh('ccache -z') - sh('make ' + target) - sh('ccache -s') - sh('make sizes') - archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf, build/**/*.bin', fingerprint: true, onlyIfSuccessful: true) - sh('make distclean') - } - } - } - } -} - -def createBuildNodeDockerLogin(String docker_repo, String docker_credentials, String target) { - return { - node { - docker.withRegistry('https://registry.hub.docker.com', docker_credentials) { - docker.image(docker_repo).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage(target) { - sh('export') - checkout scm - sh('make distclean') - sh('git fetch --tags') - sh('ccache -z') - sh('make ' + target) - sh('ccache -s') - sh('make sizes') - sh('make distclean') - } - } - } - } - } -}