Jenkins move all compile jobs to dedicated pipeline (#10149)

.ci/Jenkinsfile-compile

@@ -0,0 +1,242 @@
+#!/usr/bin/env groovy
+
+pipeline {
+  agent none
+  stages {
+
+    stage('Build') {
+      steps {
+        script {
+          def build_nodes = [:]
+          def docker_images = [
+            arch: "px4io/px4-dev-base-archlinux:2018-07-19",
+            armhf: "px4io/px4-dev-armhf:2018-07-19",
+            base: "px4io/px4-dev-base:2018-07-19",
+            clang: "px4io/px4-dev-clang:2018-07-19",
+            nuttx: "px4io/px4-dev-nuttx:2018-07-19",
+            ros: "px4io/px4-dev-ros:2018-07-19",
+            rpi: "px4io/px4-dev-raspi:2018-07-19",
+            snapdragon: "lorenzmeier/px4-dev-snapdragon:2017-12-29"
+          ]
+
+          // special builds:
+          // fmu-v2_{default, lpe} and fmu-v3_{default, rtps}
+          // bloaty compare to last successful master build
+          build_nodes["px4fmu-v2"] = {
+            node {
+              stage("Build Test px4fmu-v2") {
+                docker.image(docker_images.nuttx).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
+                  stage("px4fmu-v2") {
+                    try {
+                      checkout(scm)
+                      sh('export')
+                      sh('make distclean')
+                      sh('ccache -z')
+                      sh('git fetch --tags')
+                      sh('make nuttx_px4io-v2_default')
+                      sh('make nuttx_px4io-v2_default bloaty_symbols')
+                      sh('make nuttx_px4io-v2_default bloaty_compileunits')
+                      sh('make nuttx_px4io-v2_default bloaty_compare_master')
+                      sh('make nuttx_px4fmu-v2_default')
+                      sh('make nuttx_px4fmu-v2_default bloaty_symbols')
+                      sh('make nuttx_px4fmu-v2_default bloaty_compileunits')
+                      sh('make nuttx_px4fmu-v2_default bloaty_inlines')
+                      sh('make nuttx_px4fmu-v2_default bloaty_templates')
+                      sh('make nuttx_px4fmu-v2_default bloaty_compare_master')
+                      sh('make nuttx_px4fmu-v2_lpe')
+                      sh('make nuttx_px4fmu-v2_test')
+                      sh('make nuttx_px4fmu-v3_default')
+                      sh('make nuttx_px4fmu-v3_default bloaty_compare_master')
+                      sh('make nuttx_px4fmu-v3_rtps')
+                      sh('make sizes')
+                      sh('ccache -s')
+                      archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true)
+                    }
+                    catch (exc) {
+                      throw (exc)
+                    }
+                    finally {
+                      sh('make distclean')
+                    }
+                  }
+                }
+              }
+            }
+          }
+
+          // MAC OSX posix_sitl_default
+          build_nodes["posix_sitl_default (OSX)"] = {
+            node("mac") {
+              withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
+                stage("sitl (OSX)") {
+                  try {
+                    checkout(scm)
+                    sh('export')
+                    sh('make distclean')
+                    sh('ccache -z')
+                    sh('make posix_sitl_default')
+                    sh('ccache -s')
+                  }
+                  catch (exc) {
+                    throw (exc)
+                  }
+                  finally {
+                    sh('make distclean')
+                  }
+                }
+              }
+            }
+          }
+
+          // MAC OSX nuttx_px4fmu-v4pro_default
+          build_nodes["px4fmu-v4pro_default (OSX)"] = {
+            node("mac") {
+              withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
+                stage("px4fmu-v4pro (OSX)") {
+                  try {
+                    checkout(scm)
+                    sh('export')
+                    sh('make distclean')
+                    sh('ccache -z')
+                    sh('make nuttx_px4fmu-v4pro_default')
+                    sh('ccache -s')
+                  }
+                  catch (exc) {
+                    throw (exc)
+                  }
+                  finally {
+                    sh('make distclean')
+                  }
+                }
+              }
+            }
+          }
+
+          // docker builds:
+          def arch_builds = [
+            target: ["posix_sitl_default"],
+            image: docker_images.arch,
+            archive: false
+          ]
+
+          def armhf_builds = [
+            target: ["posix_ocpoc_ubuntu"],
+            image: docker_images.armhf,
+            archive: false
+          ]
+
+          def base_builds = [
+            target: ["posix_sitl_rtps"],
+            image: docker_images.base,
+            archive: false
+          ]
+
+          def nuttx_builds_archive = [
+            target: ["px4fmu-v4_default", "px4fmu-v4pro_default", "px4fmu-v5_default", "aerofc-v1_default",
+                     "aerocore2_default", "auav-x21_default", "crazyflie_default", "mindpx-v2_default",
+                     "nxphlite-v3_default", "tap-v1_default", "omnibus-f4sd_default"],
+            image: docker_images.nuttx,
+            archive: true
+          ]
+
+          def nuttx_builds_other = [
+            target: ["px4-same70xplained-v1_default", "px4-stm32f4discovery_default", "px4cannode-v1_default",
+                     "px4esc-v1_default", "px4nucleoF767ZI-v1_default", "s2740vc-v1_default"],
+            image: docker_images.nuttx,
+            archive: false
+          ]
+
+          def rpi_builds = [
+            target: ["posix_rpi_cross", "posix_bebop_default"],
+            image: docker_images.rpi,
+            archive: false
+          ]
+
+          def snapdragon_builds = [
+            target: ["qurt_eagle_default", "posix_eagle_default"],
+            image: docker_images.snapdragon,
+            archive: false
+          ]
+
+          def docker_builds = [
+            arch_builds, armhf_builds, base_builds, nuttx_builds_archive, nuttx_builds_other,
+            rpi_builds, snapdragon_builds
+          ]
+
+          for (def build_type = 0; build_type < docker_builds.size(); build_type++) {
+            for (def build_target = 0; build_target < docker_builds[build_type].target.size(); build_target++) {
+              build_nodes.put(docker_builds[build_type].target[build_target],
+                createBuildNode(docker_builds[build_type].archive, docker_builds[build_type].image, docker_builds[build_type].target[build_target])
+                )
+            }
+          }
+
+        parallel build_nodes
+
+        } // script
+      } // steps
+    } // stage Build
+
+    // TODO: actually upload artifacts to S3
+    stage('S3 Upload') {
+      agent {
+        docker { image 'px4io/px4-dev-base:2018-07-19' }
+      }
+      options {
+            skipDefaultCheckout()
+      }
+      when {
+        anyOf {
+          branch 'master'
+          branch 'beta'
+          branch 'stable'
+          branch 'pr-jenkins' // for testing
+        }
+      }
+      steps {
+        sh 'echo "uploading to S3"'
+      }
+    }
+
+  } // stages
+  environment {
+    CCACHE_DIR = '/tmp/ccache'
+    CI = true
+  }
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '10', artifactDaysToKeepStr: '30'))
+    timeout(time: 60, unit: 'MINUTES')
+  }
+}
+
+def createBuildNode(Boolean archive, String docker_image, String target) {
+  return {
+    node {
+      docker.withRegistry('https://registry.hub.docker.com', 'docker_hub_dagar') {
+        docker.image(docker_image).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
+          stage(target) {
+            try {
+              sh('export')
+              checkout(scm)
+              sh('make distclean')
+              sh('git fetch --tags')
+              sh('ccache -z')
+              sh('make ' + target)
+              sh('ccache -s')
+              sh('make sizes')
+              if (archive) {
+                archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf, build/**/*.bin', fingerprint: true, onlyIfSuccessful: true)
+              }
+            }
+            catch (exc) {
+              throw (exc)
+            }
+            finally {
+              sh('make distclean')
+            }
+          }
+        }
+      }
+    }
+  }
+}

Jenkinsfile

@@ -2,141 +2,6 @@ pipeline {
   agent none
   stages {
 
-    stage('Build') {
-      steps {
-        script {
-          def builds = [:]
-
-          def docker_base = "px4io/px4-dev-base:2018-07-19"
-          def docker_nuttx = "px4io/px4-dev-nuttx:2018-07-19"
-          def docker_ros = "px4io/px4-dev-ros:2018-07-19"
-          def docker_rpi = "px4io/px4-dev-raspi:2018-07-19"
-          def docker_armhf = "px4io/px4-dev-armhf:2018-07-19"
-          def docker_arch = "px4io/px4-dev-base-archlinux:2018-07-19"
-          def docker_snapdragon = "lorenzmeier/px4-dev-snapdragon:2017-12-29"
-          def docker_clang = "px4io/px4-dev-clang:2018-07-19"
-
-          // fmu-v2_{default, lpe} and fmu-v3_{default, rtps}
-          // bloaty compare to last successful master build
-          builds["px4fmu-v2"] = {
-            node {
-              stage("Build Test px4fmu-v2") {
-                docker.image(docker_nuttx).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                  stage("px4fmu-v2") {
-                    checkout scm
-                    sh "export"
-                    sh "make distclean"
-                    sh "ccache -z"
-                    sh "git fetch --tags"
-                    sh "make nuttx_px4io-v2_default"
-                    sh "make nuttx_px4io-v2_default bloaty_symbols"
-                    sh "make nuttx_px4io-v2_default bloaty_compileunits"
-                    sh "make nuttx_px4io-v2_default bloaty_compare_master"
-                    sh "make nuttx_px4fmu-v2_default"
-                    sh "make nuttx_px4fmu-v2_default bloaty_symbols"
-                    sh "make nuttx_px4fmu-v2_default bloaty_compileunits"
-                    sh "make nuttx_px4fmu-v2_default bloaty_inlines"
-                    sh "make nuttx_px4fmu-v2_default bloaty_templates"
-                    sh "make nuttx_px4fmu-v2_default bloaty_compare_master"
-                    sh "make nuttx_px4fmu-v2_lpe"
-                    sh "make nuttx_px4fmu-v2_test"
-                    sh "make nuttx_px4fmu-v3_default"
-                    sh "make nuttx_px4fmu-v3_rtps"
-                    sh "make sizes"
-                    sh "ccache -s"
-                    archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true)
-                    sh "make distclean"
-                  }
-                }
-              }
-            }
-          }
-
-          // nuttx default targets that are archived and uploaded to s3
-          for (def option in ["px4fmu-v4", "px4fmu-v4pro", "px4fmu-v5", "aerofc-v1", "aerocore2", "av-x-v1", "auav-x21", "crazyflie", "mindpx-v2", "nxphlite-v3", "tap-v1", "omnibus-f4sd"]) {
-            def node_name = "${option}"
-            builds[node_name] = createBuildNodeArchive(docker_nuttx, "${node_name}_default")
-          }
-
-          // other nuttx default targets
-          for (def option in ["px4-same70xplained-v1", "px4-stm32f4discovery", "px4cannode-v1", "px4esc-v1", "px4nucleoF767ZI-v1", "s2740vc-v1"]) {
-            def node_name = "${option}"
-            builds[node_name] = createBuildNode(docker_nuttx, "${node_name}_default")
-          }
-
-          builds["sitl_rtps"] = createBuildNode(docker_base, 'posix_sitl_rtps')
-          builds["sitl (GCC 7)"] = createBuildNode(docker_arch, 'posix_sitl_default')
-
-          builds["rpi"] = createBuildNode(docker_rpi, 'posix_rpi_cross')
-          builds["bebop"] = createBuildNode(docker_rpi, 'posix_bebop_default')
-
-          builds["ocpoc"] = createBuildNode(docker_armhf, 'posix_ocpoc_ubuntu')
-
-          // snapdragon (eagle_default)
-          builds["eagle (linux)"] = createBuildNodeDockerLogin(docker_snapdragon, 'docker_hub_dagar', 'posix_eagle_default')
-          builds["eagle (qurt)"] = createBuildNodeDockerLogin(docker_snapdragon, 'docker_hub_dagar', 'qurt_eagle_default')
-
-          // posix_sitl_default with package
-          builds["sitl"] = {
-            node {
-              stage("Build Test sitl") {
-                docker.image(docker_ros).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE') {
-                  stage("sitl") {
-                    checkout scm
-                    sh "export"
-                    sh "make distclean"
-                    sh "ccache -z"
-                    sh "make posix_sitl_default"
-                    sh "make posix_sitl_default sitl_gazebo"
-                    sh "make posix_sitl_default package"
-                    sh "ccache -s"
-                    stash name: "px4_sitl_package", includes: "build/posix_sitl_default/*.bz2"
-                    sh "make distclean"
-                  }
-                }
-              }
-            }
-          }
-
-          // MAC OS posix_sitl_default
-          builds["sitl (OSX)"] = {
-            node("mac") {
-              withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
-                stage("sitl (OSX)") {
-                  checkout scm
-                  sh "export"
-                  sh "make distclean"
-                  sh "ccache -z"
-                  sh "make posix_sitl_default"
-                  sh "ccache -s"
-                  sh "make distclean"
-                }
-              }
-            }
-          }
-
-          // MAC OS nuttx_px4fmu-v4pro_default
-          builds["px4fmu-v4pro (OSX)"] = {
-            node("mac") {
-              withEnv(["CCACHE_BASEDIR=${pwd()}"]) {
-                stage("px4fmu-v4pro (OSX)") {
-                  checkout scm
-                  sh "export"
-                  sh "make distclean"
-                  sh "ccache -z"
-                  sh "make nuttx_px4fmu-v4pro_default"
-                  sh "ccache -s"
-                  sh "make distclean"
-                }
-              }
-            }
-          }
-
-          parallel builds
-        } // script
-      } // steps
-    } // stage Builds
-
     stage('Test') {
       parallel {
 
@@ -178,6 +43,7 @@ pipeline {
               branch 'master'
               branch 'beta'
               branch 'stable'
+              branch 'pr-jenkins' // for testing
             }
           }
         }
@@ -225,6 +91,7 @@ pipeline {
               branch 'master'
               branch 'beta'
               branch 'stable'
+              branch 'pr-jenkins' // for testing
             }
           }
         }
@@ -324,70 +191,6 @@ pipeline {
           }
         }
 
-        stage('ROS MC mission box (EKF2)') {
-          agent {
-            docker {
-              image 'px4io/px4-dev-ros:2018-07-19'
-              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE'
-            }
-          }
-          options {
-            skipDefaultCheckout()
-          }
-          steps {
-            sh 'export'
-            sh 'rm -rf build; rm -rf px4-posix_sitl_default*; rm -rf .ros; rm -rf .gazebo'
-            unstash 'px4_sitl_package'
-            sh 'tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2'
-            sh 'px4-posix_sitl_default*/px4/test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=multirotor_box vehicle:=iris'
-            sh 'px4-posix_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py `find . -name *.ulg -print -quit`'
-          }
-          post {
-            always {
-              sh 'px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
-              archiveArtifacts '.ros/**/*.pdf'
-              archiveArtifacts '.ros/**/*.csv'
-              deleteDir()
-            }
-            failure {
-              sh 'ls -a'
-              archiveArtifacts '.ros/**/*.ulg'
-              archiveArtifacts '.ros/**/rosunit-*.xml'
-              archiveArtifacts '.ros/**/rostest-*.log'
-            }
-          }
-        }
-
-	stage('ROS MC mission box (LPE)') {
-          agent {
-            docker {
-              image 'px4io/px4-dev-ros:2018-07-19'
-              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE'
-            }
-          }
-          options {
-            skipDefaultCheckout()
-          }
-          steps {
-            sh 'export'
-            sh 'rm -rf build; rm -rf px4-posix_sitl_default*; rm -rf .ros; rm -rf .gazebo'
-            unstash 'px4_sitl_package'
-            sh 'tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2'
-            sh 'px4-posix_sitl_default*/px4/test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=multirotor_box vehicle:=iris est:=lpe'
-          }
-          post {
-            always {
-              sh 'px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
-              deleteDir()
-            }
-            failure {
-              sh 'ls -a'
-              archiveArtifacts '.ros/**/*.ulg'
-              archiveArtifacts '.ros/**/rosunit-*.xml'
-              archiveArtifacts '.ros/**/rostest-*.log'
-            }
-          }
-        }
 
       }
     }
@@ -450,25 +253,6 @@ pipeline {
       }
     }
 
-    // TODO: actually upload artifacts to S3
-    stage('S3 Upload') {
-      agent {
-        docker { image 'px4io/px4-dev-base:2018-07-19' }
-      }
-      options {
-            skipDefaultCheckout()
-      }
-      when {
-        anyOf {
-          branch 'master'
-          branch 'beta'
-          branch 'stable'
-        }
-      }
-      steps {
-        sh 'echo "uploading to S3"'
-      }
-    }
   } // stages
 
   environment {
@@ -480,66 +264,3 @@ pipeline {
     timeout(time: 60, unit: 'MINUTES')
   }
 }
-
-def createBuildNode(String docker_repo, String target) {
-  return {
-    node {
-      docker.image(docker_repo).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-        stage(target) {
-          sh('export')
-          checkout scm
-          sh('make distclean')
-          sh('git fetch --tags')
-          sh('ccache -z')
-          sh('make ' + target)
-          sh('ccache -s')
-          sh('make sizes')
-          sh('make distclean')
-        }
-      }
-    }
-  }
-}
-
-def createBuildNodeArchive(String docker_repo, String target) {
-  return {
-    node {
-      docker.image(docker_repo).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-        stage(target) {
-          sh('export')
-          checkout scm
-          sh('make distclean')
-          sh('git fetch --tags')
-          sh('ccache -z')
-          sh('make ' + target)
-          sh('ccache -s')
-          sh('make sizes')
-          archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/**/*.px4, build/**/*.elf, build/**/*.bin', fingerprint: true, onlyIfSuccessful: true)
-          sh('make distclean')
-        }
-      }
-    }
-  }
-}
-
-def createBuildNodeDockerLogin(String docker_repo, String docker_credentials, String target) {
-  return {
-    node {
-      docker.withRegistry('https://registry.hub.docker.com', docker_credentials) {
-        docker.image(docker_repo).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-          stage(target) {
-            sh('export')
-            checkout scm
-            sh('make distclean')
-            sh('git fetch --tags')
-            sh('ccache -z')
-            sh('make ' + target)
-            sh('ccache -s')
-            sh('make sizes')
-            sh('make distclean')
-          }
-        }
-      }
-    }
-  }
-}