Jenkins cleanup build creation

- cleanup workspace when done and increase retention

Jenkinsfile

@@ -18,46 +18,19 @@ pipeline {
         script {
           def builds = [:]
 
-
-          // nuttx default targets that are archived and uploaded to s3
-          for (def option in ["px4fmu-v4", "px4fmu-v4pro", "px4fmu-v5", "aerofc-v1"]) {
-            def node_name = "${option}"
-
-            builds["${node_name}"] = {
-              node {
-                stage("Build Test ${node_name}") {
-                  docker.image('px4io/px4-dev-nuttx:2018-03-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                    stage("${node_name}") {
-                      checkout scm
-                      sh "export"
-                      sh "make distclean"
-                      sh "ccache -z"
-                      sh "git fetch --tags"
-                      sh "make nuttx_${node_name}_default"
-                      // bloaty output and compare with last successful master
-                      sh "bloaty -n 100 -d symbols -s file build/nuttx_${node_name}_default/nuttx_${node_name}_default.elf"
-                      sh "bloaty -n 100 -d compileunits -s file build/nuttx_${node_name}_default/nuttx_${node_name}_default.elf"
-                      sh "wget --no-verbose -N https://s3.amazonaws.com/px4-travis/Firmware/master/nuttx_${node_name}_default.elf"
-                      sh "bloaty -d symbols -C full -s file build/nuttx_${node_name}_default/nuttx_${node_name}_default.elf -- nuttx_${node_name}_default.elf"
-                      sh "make nuttx_${node_name}_rtps"
-                      sh "make sizes"
-                      sh "ccache -s"
-                      archiveArtifacts(artifacts: 'build/*/*.px4', fingerprint: true)
-                      archiveArtifacts(artifacts: 'build/*/*.elf', fingerprint: true)
-                    }
-                  }
-                }
-              }
-            }
-          }
-
-
-          // special case for fmu-v2/fmu-v3
+          def docker_base = "px4io/px4-dev-base:2018-03-30"
+          def docker_nuttx = "px4io/px4-dev-nuttx:2018-03-30"
+          def docker_rpi = "px4io/px4-dev-raspi:2018-03-30"
+          def docker_armhf = "px4io/px4-dev-armhf:2017-12-30"
+          def docker_arch = "px4io/px4-dev-base-archlinux:2018-03-30"
+
+          // fmu-v2_{default, lpe} and fmu-v3_{default, rtps}
+          // bloaty compare to last successful master build
           builds["px4fmu-v2"] = {
             node {
-              stage("Build Test ${node_name}") {
-                docker.image('px4io/px4-dev-nuttx:2018-03-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                  stage("${node_name}") {
+              stage("Build Test px4fmu-v2") {
+                docker.image(docker_nuttx).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
+                  stage("px4fmu-v2") {
                     checkout scm
                     sh "export"
                     sh "make distclean"
@@ -72,178 +45,54 @@ pipeline {
                     sh "bloaty -d symbols -C full -s file build/nuttx_px4fmu-v2_default/nuttx_px4fmu-v2_default.elf -- nuttx_px4fmu-v2_default.elf"
                     sh "make nuttx_px4fmu-v2_lpe"
                     sh "make nuttx_px4fmu-v3_default"
+                    sh "wget --no-verbose -N https://s3.amazonaws.com/px4-travis/Firmware/master/nuttx_px4fmu-v3_default.elf"
+                    sh "bloaty -d symbols -C full -s file build/nuttx_px4fmu-v3_default/nuttx_px4fmu-v3_default.elf -- nuttx_px4fmu-v3_default.elf"
                     sh "make nuttx_px4fmu-v3_rtps"
                     sh "make sizes"
                     sh "ccache -s"
-                    archiveArtifacts(artifacts: 'build/*/*.px4', fingerprint: true)
-                    archiveArtifacts(artifacts: 'build/*/*.elf', fingerprint: true)
+                    archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true)
+                    sh "make distclean"
                   }
                 }
               }
             }
           }
 
-
           // nuttx default targets that are archived and uploaded to s3
-          for (def option in ["aerocore2", "auav-x21", "crazyflie", "mindpx-v2", "nxphlite-v3", "tap-v1"]) {
+          for (def option in ["px4fmu-v4", "px4fmu-v4pro", "px4fmu-v5", "aerofc-v1", "aerocore2", "auav-x21", "crazyflie", "mindpx-v2", "nxphlite-v3", "tap-v1"]) {
             def node_name = "${option}"
-
-            builds["${node_name}"] = {
-              node {
-                stage("Build Test ${node_name}") {
-                  docker.image('px4io/px4-dev-nuttx:2018-03-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                    stage("${node_name}") {
-                      checkout scm
-                      sh "export"
-                      sh "make distclean"
-                      sh "ccache -z"
-                      sh "git fetch --tags"
-                      sh "make nuttx_${node_name}_default"
-                      sh "make sizes"
-                      sh "ccache -s"
-                      archiveArtifacts(artifacts: 'build/*/*.px4', fingerprint: true)
-                      archiveArtifacts(artifacts: 'build/*/*.elf', fingerprint: true)
-                    }
-                  }
-                }
-              }
-            }
+            builds[node_name] = createBuildNode(docker_nuttx, "${node_name}_default")
           }
 
-
           // other nuttx default targets
           for (def option in ["px4-same70xplained-v1", "px4-stm32f4discovery", "px4cannode-v1", "px4esc-v1", "px4nucleoF767ZI-v1", "s2740vc-v1"]) {
             def node_name = "${option}"
-
-            builds["${node_name}"] = {
-              node {
-                stage("Build Test ${node_name}") {
-                  docker.image('px4io/px4-dev-nuttx:2018-03-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                    stage("${node_name}") {
-                      checkout scm
-                      sh "export"
-                      sh "make distclean"
-                      sh "ccache -z"
-                      sh "make nuttx_${node_name}_default"
-                      sh "make sizes"
-                      sh "ccache -s"
-                    }
-                  }
-                }
-              }
-            }
-          }
-
-
-          // posix_sitl
-          for (def option in ["sitl_default", "sitl_rtps"]) {
-            def node_name = "${option}"
-
-            builds["${node_name}"] = {
-              node {
-                stage("Build Test ${node_name}") {
-                  docker.image('px4io/px4-dev-base:2018-03-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                    stage("${node_name}") {
-                      checkout scm
-                      sh "export"
-                      sh "make distclean"
-                      sh "ccache -z"
-                      sh "make posix_${node_name}"
-                      sh "ccache -s"
-                    }
-                  }
-                }
-              }
-            }
-          }
-
-
-          // raspberry pi and bebop (armhf)
-          for (def option in ["rpi_cross", "bebop_default"]) {
-            def node_name = "${option}"
-
-            builds["${node_name}"] = {
-              node {
-                stage("Build Test ${node_name}") {
-                  docker.image('px4io/px4-dev-raspi:2018-03-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                    stage("${node_name}") {
-                      checkout scm
-                      sh "export"
-                      sh "make distclean"
-                      sh "ccache -z"
-                      sh "make posix_${node_name}"
-                      sh "ccache -s"
-                    }
-                  }
-                }
-              }
-            }
+            builds[node_name] = createBuildNode(docker_nuttx, "${node_name}_default")
           }
 
+          builds["sitl"] = createBuildNode(docker_base, "posix_sitl_default")
+          builds["sitl_rtps"] = createBuildNode(docker_base, "posix_sitl_rtps")
+          builds["sitl (GCC 7)"] = createBuildNode(docker_arch, "posix_sitl_default")
 
-          // other armhf (to be merged with raspi and bebop)
-          for (def option in ["ocpoc_ubuntu"]) {
-            def node_name = "${option}"
-
-            builds["${node_name}"] = {
-              node {
-                stage("Build Test ${node_name}") {
-                  docker.image('px4io/px4-dev-armhf:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                    stage("${node_name}") {
-                      checkout scm
-                      sh "export"
-                      sh "make distclean"
-                      sh "ccache -z"
-                      sh "make posix_${node_name}"
-                      sh "ccache -s"
-                    }
-                  }
-                }
-              }
-            }
-          }
+          builds["rpi"] = createBuildNode(docker_rpi, "posix_rpi_cross")
+          builds["bebop"] = createBuildNode(docker_rpi, "posix_bebop_default")
 
+          builds["ocpoc"] = createBuildNode(docker_armhf, "posix_ocpoc_ubuntu")
 
           // snapdragon eagle (posix + qurt)
-          for (def option in ["eagle_default"]) {
-            def node_name = "${option}"
-
-            builds["${node_name}"] = {
-              node {
-                stage("Build Test ${node_name}") {
-                  docker.withRegistry('https://registry.hub.docker.com', 'docker_hub_dagar') {
-                    docker.image("lorenzmeier/px4-dev-snapdragon:2017-12-29").inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                      stage("${node_name}") {
-                        checkout scm
-                        sh "export"
-                        sh "make distclean"
-                        sh "ccache -z"
-                        sh "make ${node_name}"
-                        sh "ccache -s"
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-
-
-          // GCC7 posix
-          for (def option in ["sitl_default"]) {
-            def node_name = "${option}"
-
-            builds["${node_name} (GCC7)"] = {
-              node {
-                stage("Build Test ${node_name} (GCC7)") {
-                  docker.image('px4io/px4-dev-base-archlinux:2018-03-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
-                    stage("${node_name}") {
+          builds["eagle"] = {
+            node {
+              stage("Build Test eagle_default") {
+                docker.withRegistry('https://registry.hub.docker.com', 'docker_hub_dagar') {
+                  docker.image("lorenzmeier/px4-dev-snapdragon:2017-12-29").inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
+                    stage("eagle_default") {
                       checkout scm
                       sh "export"
                       sh "make distclean"
                       sh "ccache -z"
-                      sh "make posix_${node_name}"
+                      sh "make eagle_default"
                       sh "ccache -s"
+                      sh "make distclean"
                     }
                   }
                 }
@@ -252,9 +101,9 @@ pipeline {
           }
 
           parallel builds
-        }
-      }
-    }
+        } // script
+      } // steps
+    } // stage Builds
 
     stage('Test') {
       parallel {
@@ -280,6 +129,7 @@ pipeline {
               reportFiles: '*',
               reportName: 'Clang Static Analyzer'
             ]
+            sh 'make distclean'
           }
           when {
             anyOf {
@@ -301,6 +151,7 @@ pipeline {
             sh 'export'
             sh 'make distclean'
             sh 'make clang-tidy-quiet'
+            sh 'make distclean'
           }
         }
 
@@ -325,6 +176,7 @@ pipeline {
               reportFiles: '*',
               reportName: 'Cppcheck'
             ]
+            sh 'make distclean'
           }
           when {
             anyOf {
@@ -347,6 +199,7 @@ pipeline {
             sh 'make distclean'
             sh 'make posix_sitl_default test_results_junit'
             junit 'build/posix_sitl_default/JUnitTestResults.xml'
+            sh 'make distclean'
           }
         }
 
@@ -371,6 +224,7 @@ pipeline {
               sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
               archiveArtifacts '.ros/**/*.pdf'
               archiveArtifacts '.ros/**/*.csv'
+              sh 'make distclean'
             }
             failure {
               archiveArtifacts '.ros/**/*.ulg'
@@ -401,6 +255,7 @@ pipeline {
               sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
               archiveArtifacts '.ros/**/*.pdf'
               archiveArtifacts '.ros/**/*.csv'
+              sh 'make distclean'
             }
             failure {
               archiveArtifacts '.ros/**/*.ulg'
@@ -431,6 +286,7 @@ pipeline {
               sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
               archiveArtifacts '.ros/**/*.pdf'
               archiveArtifacts '.ros/**/*.csv'
+              sh 'make distclean'
             }
             failure {
               archiveArtifacts '.ros/**/*.ulg'
@@ -461,6 +317,7 @@ pipeline {
               sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
               archiveArtifacts '.ros/**/*.pdf'
               archiveArtifacts '.ros/**/*.csv'
+              sh 'make distclean'
             }
             failure {
               archiveArtifacts '.ros/**/*.ulg'
@@ -470,36 +327,6 @@ pipeline {
           }
         }
 
-        stage('ROS vtol mission old 3') {
-          agent {
-            docker {
-              image 'px4io/px4-dev-ros:2018-03-30'
-              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE'
-            }
-          }
-          steps {
-            sh 'export'
-            //sh 'make distclean; rm -rf .ros; rm -rf .gazebo'
-            //sh 'git fetch --tags'
-            //sh 'make posix_sitl_default'
-            //sh 'make posix_sitl_default sitl_gazebo'
-            //sh './test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=vtol_old_3 vehicle:=standard_vtol'
-            //sh './Tools/ecl_ekf/process_logdata_ekf.py `find . -name *.ulg -print -quit`'
-          }
-          post {
-            //always {
-            //  sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
-            //  archiveArtifacts '.ros/**/*.pdf'
-            //  archiveArtifacts '.ros/**/*.csv'
-            //}
-            failure {
-              archiveArtifacts '.ros/**/*.ulg'
-              archiveArtifacts '.ros/**/rosunit-*.xml'
-              archiveArtifacts '.ros/**/rostest-*.log'
-            }
-          }
-        }
-
         stage('ROS MC mission box') {
           agent {
             docker {
@@ -521,6 +348,7 @@ pipeline {
               sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
               archiveArtifacts '.ros/**/*.pdf'
               archiveArtifacts '.ros/**/*.csv'
+              sh 'make distclean'
             }
             failure {
               archiveArtifacts '.ros/**/*.ulg'
@@ -551,6 +379,7 @@ pipeline {
               sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
               archiveArtifacts '.ros/**/*.pdf'
               archiveArtifacts '.ros/**/*.csv'
+              sh 'make distclean'
             }
             failure {
               archiveArtifacts '.ros/**/*.ulg'
@@ -581,6 +410,7 @@ pipeline {
               sh './Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg'
               archiveArtifacts '.ros/**/*.pdf'
               archiveArtifacts '.ros/**/*.csv'
+              sh 'make distclean'
             }
             failure {
               archiveArtifacts '.ros/**/*.ulg'
@@ -605,6 +435,7 @@ pipeline {
             sh 'make distclean'
             sh 'make airframe_metadata'
             archiveArtifacts(artifacts: 'airframes.md, airframes.xml', fingerprint: true)
+            sh 'make distclean'
           }
         }
 
@@ -616,6 +447,7 @@ pipeline {
             sh 'make distclean'
             sh 'make parameters_metadata'
             archiveArtifacts(artifacts: 'parameters.md, parameters.xml', fingerprint: true)
+            sh 'make distclean'
           }
         }
 
@@ -627,6 +459,7 @@ pipeline {
             sh 'make distclean'
             sh 'make module_documentation'
             archiveArtifacts(artifacts: 'modules/*.md', fingerprint: true)
+            sh 'make distclean'
           }
         }
 
@@ -642,6 +475,7 @@ pipeline {
             sh 'make distclean'
             sh 'make uorb_graphs'
             archiveArtifacts(artifacts: 'Tools/uorb_graph/graph_sitl.json')
+            sh 'make distclean'
           }
         }
       }
@@ -664,13 +498,35 @@ pipeline {
         sh 'echo "uploading to S3"'
       }
     }
-  }
+  } // stages
+
   environment {
     CCACHE_DIR = '/tmp/ccache'
     CI = true
   }
   options {
-    buildDiscarder(logRotator(numToKeepStr: '2', artifactDaysToKeepStr: '30'))
+    buildDiscarder(logRotator(numToKeepStr: '10', artifactDaysToKeepStr: '30'))
     timeout(time: 60, unit: 'MINUTES')
   }
 }
+
+def createBuildNode(String docker_repo, String target) {
+  return {
+    node {
+      docker.image(docker_repo).inside('-e CCACHE_BASEDIR=${WORKSPACE} -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') {
+        stage(target) {
+          sh('export')
+          checkout scm
+          sh('make distclean')
+          sh('git fetch --tags')
+          sh('ccache -z')
+          sh('make ' + target)
+          sh('ccache -s')
+          sh('make sizes')
+          archiveArtifacts(allowEmptyArchive: true, artifacts: 'build/**/*.px4, build/**/*.elf', fingerprint: true, onlyIfSuccessful: true)
+          sh('make distclean')
+        }
+      }
+    }
+  }
+}