zrzk
/
zr-v4
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Tools: generate_manifest: flake8 compliance

master
Peter Barker 7 years ago committed by Peter Barker
parent
02e687e99c
commit
176d26b40a
1 changed files with 92 additions and 53 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 145
      Tools/scripts/generate_manifest.py

145
Tools/scripts/generate_manifest.py
Unescape View File

@ -7,8 +7,15 @@ import json @@ -7,8 +7,15 @@ import json
import os
import re
class Firmware():
def __init__(self, date=None, platform=None, vehicletype=None, filepath=None, git_sha=None, frame=None):
def __init__(self,
date=None,
platform=None,
vehicletype=None,
filepath=None,
git_sha=None,
frame=None):
self.atts = dict()
self.atts["date"] = date
self.atts["platform"] = platform
@ -19,21 +26,24 @@ class Firmware(): @@ -19,21 +26,24 @@ class Firmware():
self.atts["release-type"] = None
self.atts["firmware-version"] = None
def __getitem__(self,what):
def __getitem__(self, what):
return self.atts[what]
def __setitem__(self,name,value):
def __setitem__(self, name, value):
self.atts[name] = value
class ManifestGenerator():
'''Return a JSON string describing "binary" directory contents under basedir'''
'''Return a JSON string describing "binary" directory contents under
basedir'''
def __init__(self,basedir,baseurl):
def __init__(self, basedir, baseurl):
self.basedir = basedir
self.baseurl = baseurl
def frame_map(self, frame):
'''translate from ArduPilot frame type terminology into mavlink terminology'''
'''translate from ArduPilot frame type terminology into mavlink
terminology'''
frame_to_mavlink_dict = {
"quad": "QUADROTOR",
"hexa": "HEXAROTOR",
@ -53,29 +63,39 @@ class ManifestGenerator(): @@ -53,29 +63,39 @@ class ManifestGenerator():
return frame
def releasetype_map(self, releasetype):
'''translate from ArduPilot release type terminology into mavlink terminology'''
if releasetype == 'stable': return 'OFFICIAL'
'''translate from ArduPilot release type terminology into mavlink
terminology'''
if releasetype == 'stable':
return 'OFFICIAL'
return releasetype.upper()
def looks_like_binaries_directory(self, dir):
'''returns True if dir looks like it is a build_binaries.py output directory'''
'''returns True if dir looks like it is a build_binaries.py output
directory'''
for entry in os.listdir(dir):
if entry in {"AntennaTracker", "Copter", "Plane", "Rover", "Sub"}:
return True
return False
def git_sha_from_git_version(self, filepath):
'''parses get-version.txt (as emitted by build_binaries.py, returns git sha from it'''
'''parses get-version.txt (as emitted by build_binaries.py, returns
git sha from it'''
content = open(filepath).read()
sha_regex = re.compile("commit (?P<sha>[0-9a-f]+)")
m = sha_regex.search(content)
if m is None:
raise Exception("filepath (%s) does not appear to contain a git sha" % (filepath,))
raise Exception(
"filepath (%s) does not contain a git sha" % (filepath,))
return m.group("sha")
def add_firmware_data_from_dir(self, dir, firmware_data, vehicletype, releasetype="dev"):
'''accumulate additional information about firmwares from a directory'''
platform_frame_regex = re.compile("(?P<board>PX4|navio|pxf)(-(?P<frame>.+))?")
def add_firmware_data_from_dir(self,
dir,
firmware_data,
vehicletype,
releasetype="dev"):
'''accumulate additional information about firmwares from directory'''
platform_frame_regex = re.compile(
"(?P<board>PX4|navio|pxf)(-(?P<frame>.+))?")
variant_firmware_regex = re.compile("[^-]+-(?P<variant>v\d+)[.px4]")
if not os.path.isdir(dir):
return
@ -87,16 +107,19 @@ class ManifestGenerator(): @@ -87,16 +107,19 @@ class ManifestGenerator():
for platformdir in dlist:
some_dir = os.path.join(dir, platformdir)
try:
git_sha = self.git_sha_from_git_version(os.path.join(some_dir, "git-version.txt"))
git_sha = self.git_sha_from_git_version(
os.path.join(some_dir, "git-version.txt"))
except Exception as e:
continue
firmware_version_file = os.path.join(some_dir, "firmware-version.txt")
firmware_version_file = os.path.join(some_dir,
"firmware-version.txt")
try:
firmware_version = open(firmware_version_file).read()
firmware_version = firmware_version.strip()
(version_numbers,release_type) = firmware_version.split("-")
(version_numbers, release_type) = firmware_version.split("-")
except ValueError as e:
# print("malformed firmware-version.txt at (%s)" % (firmware_version_file,), file=sys.stderr)
# print("malformed firmware-version.txt at (%s)" %
# (firmware_version_file,), file=sys.stderr)
firmware_version = None
except Exception as e:
# this exception is swallowed.... the current archive
@ -108,13 +131,13 @@ class ManifestGenerator(): @@ -108,13 +131,13 @@ class ManifestGenerator():
# the model type (quad/tri) is
# encoded in the platform name
# (e.g. navio-octa)
platform = m.group("board") # e.g. navio
frame = m.group("frame") # e.g. octa
platform = m.group("board") # e.g. navio
frame = m.group("frame") # e.g. octa
if frame is None:
frame = vehicletype
else:
frame = vehicletype # e.g. Plane
platform = platformdir # e.g. apm2
frame = vehicletype # e.g. Plane
platform = platformdir # e.g. apm2
for file in os.listdir(some_dir):
if file == "git-version.txt":
@ -131,32 +154,31 @@ class ManifestGenerator(): @@ -131,32 +154,31 @@ class ManifestGenerator():
# (e.g. the "v1" in
# ArduCopter-v1.px4)
variant = m.group("variant")
file_platform = "-".join([platform,variant])
file_platform = "-".join([platform, variant])
else:
file_platform = platform
firmware_format = "".join(file.split(".")[-1:])
if not vehicletype in firmware_data:
if vehicletype not in firmware_data:
firmware_data[vehicletype] = dict()
if not file_platform in firmware_data[vehicletype]:
if file_platform not in firmware_data[vehicletype]:
firmware_data[vehicletype][file_platform] = dict()
if not git_sha in firmware_data[vehicletype][file_platform]:
if git_sha not in firmware_data[vehicletype][file_platform]:
firmware_data[vehicletype][file_platform][git_sha] = dict()
if not firmware_format in firmware_data[vehicletype][file_platform][git_sha]:
firmware_data[vehicletype][file_platform][git_sha][firmware_format] = dict()
if not releasetype in firmware_data[vehicletype][file_platform][git_sha][firmware_format]:
firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype] = dict()
if not frame in firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype]:
firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype][frame] = Firmware()
sha_dict = firmware_data[vehicletype][file_platform][git_sha]
if firmware_format not in sha_dict:
sha_dict[firmware_format] = dict()
if frame not in sha_dict[firmware_format]:
sha_dict[firmware_format][frame] = Firmware()
firmware = firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype][frame]
firmware = sha_dict[firmware_format][frame]
# translate from supplied "release type" into both a
# "latest" flag andan actual release type. Also sort
# out which filepath we should use:
firmware["latest" ] = 0
firmware["latest"] = 0
if releasetype == "dev":
if firmware["filepath"] is None:
firmware["filepath"] = os.path.join(some_dir, file)
@ -185,7 +207,7 @@ class ManifestGenerator(): @@ -185,7 +207,7 @@ class ManifestGenerator():
if isinstance(xfirmwares, dict):
ret = []
for value in xfirmwares.values():
o = self.xfirmwares_to_firmwares(value)
o = self.xfirmwares_to_firmwares(value)
for oo in o:
ret.append(oo)
return ret
@ -193,24 +215,26 @@ class ManifestGenerator(): @@ -193,24 +215,26 @@ class ManifestGenerator():
return [xfirmwares]
known_release_types = {
"beta" : 1,
"latest" : 1,
"stable" : 1
"beta": 1,
"latest": 1,
"stable": 1
}
def parse_fw_version(self, version):
(version_numbers,release_type) = version.split("-")
(major,minor,patch) = version_numbers.split(".")
return (major,minor,patch,version)
(version_numbers, release_type) = version.split("-")
(major, minor, patch) = version_numbers.split(".")
return (major, minor, patch, version)
def walk_directory(self, basedir):
'''walks directory structure created by build_binaries, returns Python structure representing releases in that structure'''
'''walks directory structure created by build_binaries, returns Python
structure representing releases in that structure'''
year_month_regex = re.compile("(?P<year>\d{4})-(?P<month>\d{2})")
xfirmwares = dict()
# used to listdir basedir here, but since this is also a web document root, there's a lot of other stuff accumulated...
vehicletypes = [ 'AntennaTracker', 'Copter', 'Plane', 'Rover', 'Sub' ]
# used to listdir basedir here, but since this is also a web
# document root, there's a lot of other stuff accumulated...
vehicletypes = ['AntennaTracker', 'Copter', 'Plane', 'Rover', 'Sub']
for vehicletype in vehicletypes:
try:
vdir = os.listdir(os.path.join(basedir, vehicletype))
@ -226,12 +250,17 @@ class ManifestGenerator(): @@ -226,12 +250,17 @@ class ManifestGenerator():
# this is a dated directory e.g. binaries/Copter/2016-02
# we do not include dated directories in the manifest ATM:
continue
year_month_path = os.path.join(basedir, vehicletype, firstlevel)
year_month_path = os.path.join(basedir,
vehicletype,
firstlevel)
for fulldate in os.listdir(year_month_path):
if fulldate in ["files.html", ".makehtml"]:
# generated file which should be ignored
continue
self.add_firmware_data_from_dir(os.path.join(year_month_path, fulldate), xfirmwares, vehicletype)
self.add_firmware_data_from_dir(
os.path.join(year_month_path, fulldate),
xfirmwares,
vehicletype)
else:
# assume this is a release directory such as
# "beta", or the "latest" directory (treated as a
@ -243,7 +272,10 @@ class ManifestGenerator(): @@ -243,7 +272,10 @@ class ManifestGenerator():
tag_path = os.path.join(basedir, vehicletype, tag)
if not os.path.isdir(tag_path):
continue
self.add_firmware_data_from_dir(tag_path, xfirmwares, vehicletype, releasetype=tag)
self.add_firmware_data_from_dir(tag_path,
xfirmwares,
vehicletype,
releasetype=tag)
firmwares = self.xfirmwares_to_firmwares(xfirmwares)
@ -254,20 +286,24 @@ class ManifestGenerator(): @@ -254,20 +286,24 @@ class ManifestGenerator():
# replace the base directory with the base URL
urlifier = re.compile("^" + re.escape(basedir))
url = re.sub(urlifier, self.baseurl, filepath)
version_type = self.releasetype_map(firmware["release-type"])
some_json = dict({
"mav-autopilot": "ARDUPILOTMEGA",
# "vehicletype": firmware["vehicletype"],
# "vehicletype": firmware["vehicletype"],
"platform": firmware["platform"],
"git-sha": firmware["git_sha"],
"url": url,
"mav-type": self.frame_map(firmware["frame"]),
"mav-firmware-version-type": self.releasetype_map(firmware["release-type"]),
"mav-firmware-version-type": version_type,
"latest": firmware["latest"],
"format": firmware["format"],
})
if firmware["firmware-version"]:
(major,minor,patch,release_type) = self.parse_fw_version(firmware["firmware-version"])
some_json["mav-firmware-version"] = ".".join([major,minor,patch])
(major, minor, patch, release_type) = self.parse_fw_version(
firmware["firmware-version"])
some_json["mav-firmware-version"] = ".".join([major,
minor,
patch])
some_json["mav-firmware-version-major"] = major
some_json["mav-firmware-version-minor"] = minor
some_json["mav-firmware-version-patch"] = patch
@ -275,7 +311,7 @@ class ManifestGenerator(): @@ -275,7 +311,7 @@ class ManifestGenerator():
firmware_json.append(some_json)
ret = {
"format-version": "1.0.0", # semantic versioning
"format-version": "1.0.0", # semantic versioning
"firmware": firmware_json
}
@ -284,15 +320,18 @@ class ManifestGenerator(): @@ -284,15 +320,18 @@ class ManifestGenerator():
def json(self):
'''walk directory supplied in constructor, return json string'''
if not self.looks_like_binaries_directory(self.basedir):
print("Warning: this does not look like a binaries directory", file=sys.stderr)
print("Warning: this does not look like a binaries directory",
file=sys.stderr)
structure = self.walk_directory(self.basedir)
return json.dumps(structure, indent=4)
def usage():
return '''Usage:
generate-manifest.py basedir baseurl'''
if __name__ == "__main__":
if len(sys.argv) != 3:
print(usage())

Write Preview
Loading…
Cancel
Save