Tools/scripts/generate-manifest.py: json description of binaries directory

Tools/scripts/generate-manifest.py

@@ -0,0 +1,271 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+
+import sys
+import json
+import os
+import re
+
+class Firmware():
+    def __init__(self, date=None, platform=None, vehicletype=None, filepath=None, git_sha=None, frame=None):
+        self.atts = dict()
+        self.atts["date"] = date
+        self.atts["platform"] = platform
+        self.atts["vehicletype"] = vehicletype
+        self.atts["filepath"] = filepath
+        self.atts["git_sha"] = git_sha
+        self.atts["frame"] = frame
+        self.atts["release-type"] = None
+        self.atts["firmware-version"] = None
+
+    def __getitem__(self,what):
+        return self.atts[what]
+
+    def __setitem__(self,name,value):
+        self.atts[name] = value
+
+class ManifestGenerator():
+    '''Return a JSON string describing "binary" directory contents under basedir'''
+
+    def __init__(self,basedir,baseurl):
+        self.basedir = basedir
+        self.baseurl = baseurl
+
+    def frame_map(self, frame):
+        '''translate from ArduPilot frame type terminology into mavlink terminology'''
+        frame_to_mavlink_dict = {
+            "quad": "QUADROTOR",
+            "hexa": "HEXAROTOR",
+            "y6": "ARDUPILOT_Y6",
+            "tri": "TRICOPTER",
+            "octa": "OCTOROTOR",
+            "octa-quad": "ARDUPILOT_OCTAQUAD",
+            "heli": "HELICOPTER",
+            "Plane": "FIXED_WING",
+            "AntennaTracker": "ANTENNA_TRACKER",
+            "Rover": "GROUND_ROVER",
+            "PX4IO": "ARDUPILOT_PX4IO",
+        }
+        if frame in frame_to_mavlink_dict:
+            return frame_to_mavlink_dict[frame]
+
+        return frame
+
+    def releasetype_map(self, releasetype):
+        '''translate from ArduPilot release type terminology into mavlink terminology'''
+        if releasetype == 'stable': return 'OFFICIAL'
+        return releasetype.upper()
+
+    def looks_like_binaries_directory(self, dir):
+        '''returns True if dir looks like it is a build_binaries.sh output directory'''
+        for entry in os.listdir(dir):
+            if entry in {"AntennaTracker", "Copter", "Plane", "Rover"}:
+                return True
+        return False
+
+    def git_sha_from_git_version(self, filepath):
+        '''parses get-version.txt (as emitted by build_binaries.sh, returns git sha from it'''
+        content = open(filepath).read()
+        sha_regex = re.compile("commit (?P<sha>[0-9a-f]+)")
+        m = sha_regex.search(content)
+        if m is None:
+            raise Exception("filepath (%s) does not appear to contain a git sha" % (filepath,))
+        return m.group("sha")
+
+    def add_firmware_data_from_dir(self, dir, firmware_data, vehicletype, releasetype="dev"):
+        '''accumulate additional information about firmwares from a directory'''
+        platform_frame_regex = re.compile("(?P<board>PX4|navio|pxf)(-(?P<frame>.+))?")
+        variant_firmware_regex = re.compile("[^-]+-(?P<variant>v\d+)[.px4]")
+        for platformdir in os.listdir(dir):
+            some_dir = os.path.join(dir, platformdir)
+            try:
+                git_sha = self.git_sha_from_git_version(os.path.join(some_dir, "git-version.txt"))
+            except Exception as e:
+                continue
+            try:
+                firmware_version = open(os.path.join(some_dir, "firmware-version.txt")).read()
+                firmware_version = firmware_version.strip()
+            except Exception as e:
+                # this exception is swallowed.... the current archive
+                # is incomplete.
+                firmware_version = None
+
+            m = platform_frame_regex.match(platformdir)
+            if m is not None:
+                # the model type (quad/tri) is
+                # encoded in the platform name
+                # (e.g. navio-octa)
+                platform = m.group("board") # e.g. navio
+                frame = m.group("frame") # e.g. octa
+                if frame is None:
+                    frame = vehicletype
+            else:
+                frame = vehicletype # e.g. Plane
+                platform = platformdir # e.g. apm2
+
+            for file in os.listdir(some_dir):
+                if file == "git-version.txt":
+                    continue
+                if file == "firmware-version.txt":
+                    continue
+
+                m = variant_firmware_regex.match(file)
+                if m:
+                    # the platform variant is
+                    # encoded in the firmware filename
+                    # (e.g. the "v1" in
+                    # ArduCopter-v1.px4)
+                    variant = m.group("variant")
+                    file_platform = "-".join([platform,variant])
+                else:
+                    file_platform = platform
+
+                firmware_format = "".join(file.split(".")[-1:])
+
+                if not vehicletype in firmware_data:
+                    firmware_data[vehicletype] = dict()
+                if not file_platform in firmware_data[vehicletype]:
+                    firmware_data[vehicletype][file_platform] = dict()
+                if not git_sha in firmware_data[vehicletype][file_platform]:
+                    firmware_data[vehicletype][file_platform][git_sha] = dict()
+
+                if not firmware_format in firmware_data[vehicletype][file_platform][git_sha]:
+                    firmware_data[vehicletype][file_platform][git_sha][firmware_format] = dict()
+                if not frame in firmware_data[vehicletype][file_platform][git_sha][firmware_format]:
+                    firmware_data[vehicletype][file_platform][git_sha][firmware_format][frame] = Firmware()
+
+                firmware = firmware_data[vehicletype][file_platform][git_sha][firmware_format][frame]
+
+                # translate from supplied "release type" into both a
+                # "latest" flag andan actual release type.  Also sort
+                # out which filepath we should use:
+                firmware["latest" ] = 0
+                if releasetype == "dev":
+                    if firmware["filepath"] is None:
+                        firmware["filepath"] = os.path.join(some_dir, file)
+                    if firmware["release-type"] is None:
+                        firmware["release-type"] = "dev"
+                elif releasetype == "latest":
+                    firmware["latest"] = 1
+                    firmware["filepath"] = os.path.join(some_dir, file)
+                    if firmware["release-type"] is None:
+                        firmware["release-type"] = "dev"
+                else:
+                    if (not firmware["latest"]):
+                        firmware["filepath"] = os.path.join(some_dir, file)
+                    firmware["release-type"] = releasetype
+
+                firmware["platform"] = file_platform
+                firmware["vehicletype"] = vehicletype
+                firmware["git_sha"] = git_sha
+                firmware["frame"] = frame
+                firmware["timestamp"] = os.path.getctime(firmware["filepath"])
+                firmware["format"] = firmware_format
+                firmware["firmware-version"] = firmware_version
+
+    def xfirmwares_to_firmwares(self, xfirmwares):
+        '''takes hash structure of firmwares, returns list of them'''
+        if isinstance(xfirmwares, dict):
+            ret = []
+            for value in xfirmwares.values():
+                o  = self.xfirmwares_to_firmwares(value)
+                for oo in o:
+                    ret.append(oo)
+            return ret
+        else:
+            return [xfirmwares]
+
+    known_release_types = {
+        "beta" : 1,
+        "latest" : 1,
+        "stable" : 1
+    }
+
+    def parse_fw_version(self, version):
+        (version_numbers,release_type) = version.split("-")
+        (major,minor,patch) = version_numbers.split(".")
+        return (major,minor,patch,version)
+
+    def walk_directory(self, basedir):
+        '''walks directory structure created by build_binaries, returns Python structure representing releases in that structure'''
+        year_month_regex = re.compile("(?P<year>\d{4})-(?P<month>\d{2})")
+
+        xfirmwares = dict()
+
+        # used to listdir basedir here, but since this is also a web document root, there's a lot of other stuff accumulated...
+        vehicletypes = [ 'AntennaTracker', 'Copter', 'Plane', 'PX4IO', 'Rover' ]
+        for vehicletype in vehicletypes:
+            vdir = os.listdir(os.path.join(basedir, vehicletype))
+            for firstlevel in vdir:
+                if year_month_regex.match(firstlevel):
+                    # this is a dated directory e.g. binaries/Copter/2016-02
+                    year_month_path = os.path.join(basedir, vehicletype, firstlevel)
+                    for fulldate in os.listdir(year_month_path):
+                        self.add_firmware_data_from_dir(os.path.join(year_month_path, fulldate), xfirmwares, vehicletype)
+                else:
+                    # assume this is a release directory such as
+                    # "beta", or the "latest" directory (treated as a
+                    # release and handled specially later)
+                    tag = firstlevel
+                    if tag not in self.known_release_types:
+                        print("Unknown tag (%s) in directory (%s)" %
+                              (tag, vdir))
+                    tag_path = os.path.join(basedir, vehicletype, tag)
+                    self.add_firmware_data_from_dir(tag_path, xfirmwares, vehicletype, releasetype=tag)
+
+        firmwares = self.xfirmwares_to_firmwares(xfirmwares)
+
+        # convert from ardupilot-naming conventions to common JSON format:
+        firmware_json = []
+        for firmware in firmwares:
+            filepath = firmware["filepath"]
+            # replace the base directory with the base URL
+            urlifier = re.compile("^" + re.escape(basedir))
+            url = re.sub(urlifier, self.baseurl, filepath)
+            some_json = dict({
+                "mav-autopilot": "ARDUPILOTMEGA",
+#                "vehicletype": firmware["vehicletype"],
+                "platform": firmware["platform"],
+                "git-sha": firmware["git_sha"],
+                "url": url,
+                "mav-type": self.frame_map(firmware["frame"]),
+                "mav-firmware-version-type": self.releasetype_map(firmware["release-type"]),
+                "latest": firmware["latest"],
+                "format": firmware["format"],
+            })
+            if firmware["firmware-version"]:
+                (major,minor,patch,release_type) = self.parse_fw_version(firmware["firmware-version"])
+                some_json["mav-firmware-version"] = ".".join([major,minor,patch])
+                some_json["mav-firmware-version-major"] = major
+                some_json["mav-firmware-version-minor"] = minor
+                some_json["mav-firmware-version-patch"] = patch
+
+            firmware_json.append(some_json)
+
+        ret = {
+            "format-version": "1.0.0", # semantic versioning
+            "firmware": firmware_json
+        }
+
+        return ret
+
+    def json(self):
+        '''walk directory supplied in constructor, return json string'''
+        if not self.looks_like_binaries_directory(self.basedir):
+            print("Warning: this does not look like a binaries directory", file=sys.stderr)
+
+        structure = self.walk_directory(self.basedir)
+        return json.dumps(structure, indent=4)
+
+def usage():
+    return '''Usage:
+generate-manifest.py basedir baseurl'''
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print(usage())
+        sys.exit(1)
+
+    generator = ManifestGenerator(sys.argv[1], sys.argv[2])
+    print(generator.json())