Tools: update build scripts

Tools/scripts/build_binaries.py

@@ -20,12 +20,15 @@ import gzip
 
 # local imports
 import generate_manifest, gen_stable
-
+import build_binaries_history
 
 class build_binaries(object):
     def __init__(self, tags):
         self.tags = tags
         self.dirty = False
+        binaries_history_filepath = os.path.join(self.buildlogs_dirpath(),
+                                                 "build_binaries_history.sqlite")
+        self.history = build_binaries_history.BuildBinariesHistory(binaries_history_filepath)
 
     def progress(self, string):
         '''pretty-print progress'''
@@ -114,7 +117,11 @@ is bob we will attempt to checkout bob-AVR'''
         if ctag == "latest":
             vtag = "master"
         else:
-            vtag = "%s-%s" % (vehicle, ctag)
+            tagvehicle = vehicle
+            if tagvehicle == "Rover":
+                # FIXME: Rover tags in git still named APMrover2 :-(
+                tagvehicle = "APMrover2"
+            vtag = "%s-%s" % (tagvehicle, ctag)
 
         branches = []
         if cframe is not None:
@@ -331,9 +338,13 @@ is bob we will attempt to checkout bob-AVR'''
         self.progress("Building %s %s binaries (cwd=%s)" %
                       (vehicle, tag, os.getcwd()))
 
-        for board in boards:
+        board_count = len(boards)
+        count = 0
+        for board in sorted(boards, key=str.lower):
             now = datetime.datetime.now()
-            self.progress("Building board: %s at %s" % (board, str(now)))
+            count += 1
+            self.progress("[%u/%u] Building board: %s at %s" %
+                          (count, board_count, board, str(now)))
             for frame in frames:
                 if frame is not None:
                     self.progress("Considering frame %s for board %s" %
@@ -373,6 +384,8 @@ is bob we will attempt to checkout bob-AVR'''
 
                 self.remove_tmpdir()
 
+                githash = self.run_git(["rev-parse", "HEAD"]).rstrip()
+
                 t0 = time.time()
 
                 self.progress("Configuring for %s in %s" %
@@ -396,11 +409,16 @@ is bob we will attempt to checkout bob-AVR'''
                            (vehicle, board, framesuffix, tag))
                     self.progress(msg)
                     self.error_strings.append(msg)
+                    # record some history about this build
+                    t1 = time.time()
+                    time_taken_to_build = t1-t0
+                    self.history.record_build(githash, tag, vehicle, board, frame, None, t0, time_taken_to_build)
                     continue
 
                 t1 = time.time()
+                time_taken_to_build = t1-t0
                 self.progress("Building %s %s %s %s took %u seconds" %
-                              (vehicle, tag, board, frame, t1-t0))
+                              (vehicle, tag, board, frame, time_taken_to_build))
 
                 bare_path = os.path.join(self.buildroot,
                                          board,
@@ -415,8 +433,10 @@ is bob we will attempt to checkout bob-AVR'''
                     filepath = "".join([bare_path, extension])
                     if os.path.exists(filepath):
                         files_to_copy.append(filepath)
+                if not os.path.exists(bare_path):
+                    raise Exception("No elf file?!")
                 # only copy the elf if we don't have other files to copy
-                if os.path.exists(bare_path) and len(files_to_copy) == 0:
+                if len(files_to_copy) == 0:
                     files_to_copy.append(bare_path)
 
                 for path in files_to_copy:
@@ -428,6 +448,9 @@ is bob we will attempt to checkout bob-AVR'''
                 self.touch_filepath(os.path.join(self.binaries,
                                                  vehicle_binaries_subdir, tag))
 
+                # record some history about this build
+                self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build)
+
         if not self.checkout(vehicle, tag, "PX4", None):
             self.checkout(vehicle, "latest")
             return
@@ -541,6 +564,7 @@ is bob we will attempt to checkout bob-AVR'''
                 "MatekF405-STD",
                 "MatekF405-Wing",
                 "MatekF765-Wing",
+                "MatekH743",
                 "OMNIBUSF7V2",
                 "sparky2",
                 "omnibusf4",
@@ -559,13 +583,17 @@ is bob we will attempt to checkout bob-AVR'''
                 "PH4-mini",
                 "CUAVv5",
                 "CUAVv5Nano",
+                "CUAV-Nora",
+                "CUAV-X7",
                 "mRoX21",
                 "Pixracer",
                 "F4BY",
                 "mRoX21-777",
                 "mRoControlZeroF7",
+                "mRoNexus",
                 "F35Lightning",
                 "speedybeef4",
+                "SuccexF4",
                 "DrotekP3Pro",
                 "VRBrain-v51",
                 "VRBrain-v52",
@@ -576,6 +604,7 @@ is bob we will attempt to checkout bob-AVR'''
                 "Durandal",
                 "CubeOrange",
                 "CubeYellow",
+                "R9Pilot",
                 # SITL targets
                 "SITL_x86_64_linux_gnu",
                 "SITL_arm_linux_gnueabihf",
@@ -586,6 +615,10 @@ is bob we will attempt to checkout bob-AVR'''
         return ["f103-GPS",
                 "f103-ADSB",
                 "f103-RangeFinder",
+                "f303-GPS",
+                "f303-Universal",
+                "f303-M10025",
+                "f303-M10070",
                 "CUAV_GPS",
                 "ZubaxGNSS",
                 ]
@@ -628,11 +661,11 @@ is bob we will attempt to checkout bob-AVR'''
         '''build Rover binaries'''
         boards = self.common_boards()
         self.build_vehicle(tag,
-                           "APMrover2",
+                           "Rover",
                            boards,
                            "Rover",
                            "ardurover",
-                           "APMrover2")
+                           "Rover")
 
     def build_ardusub(self, tag):
         '''build Sub binaries'''
@@ -657,7 +690,7 @@ is bob we will attempt to checkout bob-AVR'''
     def generate_manifest(self):
         '''generate manigest files for GCS to download'''
         self.progress("Generating manifest")
-        base_url = 'http://firmware.ardupilot.org'
+        base_url = 'https://firmware.ardupilot.org'
         generator = generate_manifest.ManifestGenerator(self.binaries,
                                                         base_url)
         content = generator.json()
@@ -756,12 +789,14 @@ is bob we will attempt to checkout bob-AVR'''
                                       "binaries.build")
 
         for tag in self.tags:
+            t0 = time.time()
             self.build_arducopter(tag)
             self.build_arduplane(tag)
             self.build_rover(tag)
             self.build_antennatracker(tag)
             self.build_ardusub(tag)
             self.build_AP_Periph(tag)
+            self.history.record_run(githash, tag, t0, time.time()-t0)
 
         if os.path.exists(self.tmpdir):
             shutil.rmtree(self.tmpdir)

Tools/scripts/configure_all.py

@@ -16,21 +16,27 @@ parser.add_argument('--build', action='store_true', default=False, help='build a
 parser.add_argument('--build-target', default='copter', help='build target')
 parser.add_argument('--stop', action='store_true', default=False, help='stop on build fail')
 parser.add_argument('--no-bl', action='store_true', default=False, help="don't check bootloader builds")
+parser.add_argument('--Werror', action='store_true', default=False, help="build with -Werror")
 parser.add_argument('--pattern', default='*')
+parser.add_argument('--start', default=None, type=int, help='continue from specified build number')
 parser.add_argument('--python', default='python')
 args = parser.parse_args()
 
 os.environ['PYTHONUNBUFFERED'] = '1'
 
 failures = []
+done = []
+board_list = []
 
 def get_board_list():
     '''add boards based on existance of hwdef-bl.dat in subdirectories for ChibiOS'''
     board_list = []
+    # these are base builds, and don't build directly
+    omit = ['f103-periph', 'f303-periph']
     dirname, dirlist, filenames = next(os.walk('libraries/AP_HAL_ChibiOS/hwdef'))
     for d in dirlist:
         hwdef = os.path.join(dirname, d, 'hwdef.dat')
-        if os.path.exists(hwdef):
+        if os.path.exists(hwdef) and not d in omit:
             board_list.append(d)
     return board_list
 
@@ -38,17 +44,30 @@ def run_program(cmd_list, build):
     print("Running (%s)" % " ".join(cmd_list))
     retcode = subprocess.call(cmd_list)
     if retcode != 0:
-        print("Build failed: %s %s" % (build, ' '.join(cmd_list)))
+        print("FAILED BUILD: %s %s" % (build, ' '.join(cmd_list)))
         global failures
         failures.append(build)
         if args.stop:
             sys.exit(1)
 
-for board in get_board_list():
+for board in sorted(get_board_list()):
     if not fnmatch.fnmatch(board, args.pattern):
         continue
-    print("Configuring for %s" % board)
-    run_program([args.python, "waf", "configure", "--board", board], "configure: " + board)
+    board_list.append(board)
+
+if args.start is not None:
+    if args.start < 1 or args.start >= len(board_list):
+        print("Invalid start %u for %u boards" % (args.start, len(board_list)))
+        sys.exit(1)
+    board_list = board_list[args.start-1:]
+
+for board in board_list:
+    done.append(board)
+    print("Configuring for %s [%u/%u failed=%u]" % (board, len(done), len(board_list), len(failures)))
+    config_opts = ["--board", board]
+    if args.Werror:
+        config_opts += ["--Werror"]
+    run_program([args.python, "waf", "configure"] + config_opts, "configure: " + board)
     if args.build:
         if board == "iomcu":
             target = "iofirmware"
@@ -56,7 +75,10 @@ for board in get_board_list():
             target = "AP_Periph"
         else:
             target = args.build_target
-        run_program([args.python, "waf", target], "build: " + board)
+        if target.find('/') != -1:
+            run_program([args.python, "waf", "--target", target], "build: " + board)
+        else:
+            run_program([args.python, "waf", target], "build: " + board)
     if args.no_bl:
         continue
     # check for bootloader def

Tools/scripts/decode_devid.py

@@ -93,6 +93,13 @@ if opts.compass:
 if opts.imu:
     decoded_devname = imu_types.get(devtype, "UNKNOWN")
 
-print("bus_type:%s(%u)  bus:%u address:%u(0x%x) devtype:%u(0x%x) %s" % (
-    bustypes.get(bus_type,"UNKNOWN"), bus_type,
-    bus, address, address, devtype, devtype, decoded_devname))
+
+if bus_type == 3:
+    #uavcan devtype represents sensor_id
+    print("bus_type:%s(%u)  bus:%u address:%u(0x%x) sensor_id:%u(0x%x) %s" % (
+        bustypes.get(bus_type,"UNKNOWN"), bus_type,
+        bus, address, address, devtype-1, devtype-1, decoded_devname))
+else:
+    print("bus_type:%s(%u)  bus:%u address:%u(0x%x) devtype:%u(0x%x) %s" % (
+        bustypes.get(bus_type,"UNKNOWN"), bus_type,
+        bus, address, address, devtype, devtype, decoded_devname))

Tools/scripts/generate_manifest.py

@@ -20,6 +20,7 @@ brand_map = {
     'PH4-mini' : ('Pixhawk 4 Mini', 'Holybro'),
     'KakuteF4' : ('KakuteF4', 'Holybro'),
     'KakuteF7' : ('KakuteF7', 'Holybro'),
+    'KakuteF7Mini' : ('KakuteF7Mini', 'Holybro'),
     'CubeBlack' : ('CubeBlack', 'Hex/ProfiCNC'),
     'CubeYellow' : ('CubeYellow', 'Hex/ProfiCNC'),
     'CubeOrange' : ('CubeOrange', 'Hex/ProfiCNC'),
@@ -28,6 +29,8 @@ brand_map = {
     'CubeGreen-solo' : ('CubeGreen Solo', 'Hex/ProfiCNC'),
     'CUAVv5' : ('CUAVv5', 'CUAV'),
     'CUAVv5Nano' : ('CUAVv5 Nano', 'CUAV'),
+    'CUAV-Nora' : ('CUAV Nora', 'CUAV'),
+    'CUAV-X7' : ('CUAV X7', 'CUAV'),
     'DrotekP3Pro' : ('Pixhawk 3 Pro', 'Drotek'),
     'MatekF405' : ('Matek F405', 'Matek'),
     'MatekF405-STD' : ('Matek F405 STD', 'Matek'),
@@ -37,6 +40,7 @@ brand_map = {
     'Pixracer' : ('PixRacer', 'mRobotics'),
     'mRoX21' : ('mRo X2.1', 'mRobotics'),
     'mRoX21-777' : ('mRo X2.1-777', 'mRobotics'),
+    'mRoNexus' : ('mRo Nexus', 'mRobotics'),
     'TBS-Colibri-F7' : ('Colibri F7', 'TBS'),
     'sparky2' : ('Sparky2', 'TauLabs'),
     'mindpx-v2' : ('MindPX V2', 'AirMind'),
@@ -188,7 +192,7 @@ class ManifestGenerator():
         }
         if 'USBID' in apj_json:
             # newer APJ files have USBID in the json data
-            firmware['USBID'] = apj_json['USBID']
+            firmware['USBID'] = [apj_json['USBID']]
         elif platform in USBID_MAP:
             firmware['USBID'] = USBID_MAP[platform]
         else:
@@ -224,6 +228,11 @@ class ManifestGenerator():
             (brand_name, manufacturer) = brand_map[platform]
             firmware['brand_name'] = brand_name
             firmware['manufacturer'] = manufacturer
+        # copy over some extra information if available
+        extra_tags = [ 'image_size' ]
+        for tag in extra_tags:
+            if tag in apj_json:
+                firmware[tag] = apj_json[tag]
 
     def add_USB_IDs(self, firmware):
         '''add USB IDs to a firmware'''
@@ -329,7 +338,7 @@ class ManifestGenerator():
 
                 filepath = os.path.join(some_dir, filename)
                 firmware_format = self.firmware_format_for_filepath(filepath)
-                if firmware_format not in [ "ELF", "abin", "apj", "hex", "px4" ]:
+                if firmware_format not in [ "ELF", "abin", "apj", "hex", "px4", "bin" ]:
                     print("Unknown firmware format (%s)" % firmware_format)
 
                 firmware = Firmware()
@@ -471,7 +480,7 @@ class ManifestGenerator():
                   file=sys.stderr)
 
         structure = self.walk_directory(self.basedir)
-        return json.dumps(structure, indent=4)
+        return json.dumps(structure, indent=4, separators=(',', ': '))
 
 
 def usage():
@@ -484,7 +493,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='generate manifest.json')
 
     parser.add_argument('--outfile', type=str, default=None, help='output file, default stdout')
-    parser.add_argument('--baseurl', type=str, default="http://firmware.ardupilot.org", help='base binaries directory')
+    parser.add_argument('--baseurl', type=str, default="https://firmware.ardupilot.org", help='base binaries directory')
     parser.add_argument('basedir', type=str, default="-", help='base binaries directory')
 
     args = parser.parse_args()