From f528c5d2060f7112969d7e607e77c713bc9c8880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beat=20K=C3=BCng?= Date: Mon, 22 Mar 2021 11:58:22 +0100 Subject: [PATCH] github ci: push parameter metadata to s3 --- .github/workflows/deploy_all.yml | 77 ++++++++++++++++ .github/workflows/metadata.yml | 20 ++-- Makefile | 2 +- Tools/generate_board_targets_json.py | 92 +++++++++++++++++++ .../version/get_git_tag_or_branch_version.sh | 21 +++++ src/lib/version/px_update_git_header.py | 51 +++++----- 6 files changed, 235 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/deploy_all.yml create mode 100755 Tools/generate_board_targets_json.py create mode 100755 src/lib/version/get_git_tag_or_branch_version.sh diff --git a/.github/workflows/deploy_all.yml b/.github/workflows/deploy_all.yml new file mode 100644 index 0000000000..51a7a859bb --- /dev/null +++ b/.github/workflows/deploy_all.yml @@ -0,0 +1,77 @@ +name: Deploy metadata for all targets + +on: + push: + branches: + - 'master' + - 'release/*' + - 'pr-metadata-test' + +jobs: + enumerate_targets: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v1 + with: + token: ${{secrets.ACCESS_TOKEN}} + - id: set-matrix + run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)" + build: + runs-on: ubuntu-latest + needs: enumerate_targets + strategy: + matrix: ${{fromJson(needs.enumerate_targets.outputs.matrix)}} + container: px4io/px4-dev-${{ matrix.container }}:2021-02-04 + steps: + - uses: actions/checkout@v1 + with: + token: ${{secrets.ACCESS_TOKEN}} + + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: cmake -P {0} + run: | + string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) + message("::set-output name=timestamp::${current_date}") + - name: ccache cache files + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ${{matrix.target}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} + restore-keys: ${{matrix.target}}-ccache- + - name: setup ccache + run: | + mkdir -p ~/.ccache + echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf + echo "compression = true" >> ~/.ccache/ccache.conf + echo "compression_level = 5" >> ~/.ccache/ccache.conf + echo "max_size = 100M" >> ~/.ccache/ccache.conf + ccache -s + ccache -z + + - name: make ${{matrix.target}} + run: make ${{matrix.target}} + - name: ccache post-run + run: ccache -s + + - name: parameter metadata + run: | + make ${{matrix.target}} ver_gen + ./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV + cd build/${{ matrix.target }} + mkdir _metadata || true + cp parameters.* _metadata + + - uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read + env: + AWS_S3_BUCKET: 'px4-travis' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-west-1' + SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/' + DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/' + diff --git a/.github/workflows/metadata.yml b/.github/workflows/metadata.yml index d73eff4ae0..0c022860f1 100644 --- a/.github/workflows/metadata.yml +++ b/.github/workflows/metadata.yml @@ -4,9 +4,8 @@ on: push: branches: - 'master' - pull_request: - branches: - - '*' + - 'release/*' + - 'pr-metadata-test' jobs: @@ -51,9 +50,18 @@ jobs: - name: parameter metadata run: | make parameters_metadata - cd build/px4_sitl_default/docs - ls -ls * - # TODO: deploy 'parameters.md, parameters.xml to S3 px4-travis:Firmware/master/ + ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV + + - uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read + env: + AWS_S3_BUCKET: 'px4-travis' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-west-1' + SOURCE_DIR: 'build/px4_sitl_default/docs/' + DEST_DIR: 'Firmware/${{ env.version }}/_general/' uorb_graph: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 3078b79190..99e8657960 100644 --- a/Makefile +++ b/Makefile @@ -318,7 +318,7 @@ coverity_scan: px4_sitl_default .PHONY: parameters_metadata airframe_metadata module_documentation px4_metadata doxygen parameters_metadata: - @$(MAKE) --no-print-directory px4_sitl_default metadata_parameters + @$(MAKE) --no-print-directory px4_sitl_default metadata_parameters ver_gen airframe_metadata: @$(MAKE) --no-print-directory px4_sitl_default metadata_airframes diff --git a/Tools/generate_board_targets_json.py b/Tools/generate_board_targets_json.py new file mode 100755 index 0000000000..53c02c3fe8 --- /dev/null +++ b/Tools/generate_board_targets_json.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" Script to generate a JSON config with all build targets (for CI) """ + +import argparse +import os +import sys +import json +import re + +source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') + +parser = argparse.ArgumentParser(description='Generate build targets') + +parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='Verbose Output') +parser.add_argument('-p', '--pretty', dest='pretty', action='store_true', + help='Pretty output instead of a single line') + +args = parser.parse_args() +verbose = args.verbose + +build_configs = [] +excluded_manufacturers = ['atlflight'] +excluded_platforms = ['qurt'] +excluded_labels = ['stackcheck', 'nolockstep', 'replay', 'test'] + +def process_target(cmake_file, target_name): + ret = None + is_board_def = False + platform = None + toolchain = None + for line in open(cmake_file, 'r'): + if 'px4_add_board' in line: + is_board_def = True + if not is_board_def: + continue + + re_platform = re.search('PLATFORM\s+([^\s]+)', line) + if re_platform: platform = re_platform.group(1) + + re_toolchain = re.search('TOOLCHAIN\s+([^\s]+)', line) + if re_toolchain: toolchain = re_toolchain.group(1) + + if is_board_def: + assert platform, f"PLATFORM not found in {cmake_file}" + + if platform not in excluded_platforms: + # get the container based on the platform and toolchain + container = platform + if platform == 'posix': + container = 'base-focal' + if toolchain: + if toolchain.startswith('aarch64'): + container = 'aarch64' + elif toolchain == 'arm-linux-gnueabihf': + container = 'armhf' + else: + if verbose: print(f'possibly unmatched toolchain: {toolchain}') + elif platform == 'nuttx': + container = 'nuttx-focal' + + ret = {'target': target_name, 'container': container} + return ret + +for manufacturer in os.scandir(os.path.join(source_dir, 'boards')): + if not manufacturer.is_dir(): + continue + if manufacturer.name in excluded_manufacturers: + if verbose: print(f'excluding manufacturer {manufacturer.name}') + continue + + for board in os.scandir(manufacturer.path): + if not board.is_dir(): + continue + for files in os.scandir(board.path): + if files.is_file() and files.name.endswith('.cmake'): + label = files.name[:-6] + target_name = manufacturer.name + '_' + board.name + '_' + label + if label in excluded_labels: + if verbose: print(f'excluding label {label} ({target_name})') + continue + target = process_target(files.path, target_name) + if target is not None: + build_configs.append(target) + + +github_action_config = { 'include': build_configs } +extra_args = {} +if args.pretty: + extra_args['indent'] = 2 +print(json.dumps(github_action_config, **extra_args)) + diff --git a/src/lib/version/get_git_tag_or_branch_version.sh b/src/lib/version/get_git_tag_or_branch_version.sh new file mode 100755 index 0000000000..a7d9c13b33 --- /dev/null +++ b/src/lib/version/get_git_tag_or_branch_version.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +# Script to extract PX4_GIT_TAG_OR_BRANCH_NAME to be used by CI + +build_dir="$1" +if [ ! -d "$build_dir" ]; then + echo "usage: $0 " + exit -1 +fi + +version_file="$build_dir/src/lib/version/build_git_version.h" + +if [ ! -f "$version_file" ]; then + echo "Version file not found. Did you run the target 'ver_gen'?" + exit -1 +fi + +sed -n 's/.*PX4_GIT_TAG_OR_BRANCH_NAME\s*"\(.*\)".*/version=\1/p' "$version_file" + diff --git a/src/lib/version/px_update_git_header.py b/src/lib/version/px_update_git_header.py index 42c00676c0..e9372ff36b 100755 --- a/src/lib/version/px_update_git_header.py +++ b/src/lib/version/px_update_git_header.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function import argparse import os @@ -37,8 +36,17 @@ header = """ # PX4 -git_tag = subprocess.check_output('git describe --exclude ext/* --always --tags --dirty'.split(), +git_describe_cmd = 'git describe --exclude ext/* --always --tags --dirty' +git_tag = subprocess.check_output(git_describe_cmd.split(), stderr=subprocess.STDOUT).decode('utf-8').strip() + +try: + # get the tag if we're on a tagged commit + tag_or_branch = subprocess.check_output((git_describe_cmd+' --exact-match').split(), + stderr=subprocess.STDOUT).decode('utf-8').strip() +except: + tag_or_branch = None + if validate: if verbose: print("testing git tag: "+git_tag) @@ -92,18 +100,23 @@ try: except: oem_tag = '' -header += """ -#define PX4_GIT_VERSION_STR "{git_version}" +if tag_or_branch is None: + # replace / so it can be used as directory name + tag_or_branch = git_branch_name.replace('/', '-') + # either a release or master branch (used for metadata) + if not tag_or_branch.startswith('release-'): + tag_or_branch = 'master' + +header += f""" +#define PX4_GIT_VERSION_STR "{git_version}" #define PX4_GIT_VERSION_BINARY 0x{git_version_short} -#define PX4_GIT_TAG_STR "{git_tag}" -#define PX4_GIT_BRANCH_NAME "{git_branch_name}" +#define PX4_GIT_TAG_STR "{git_tag}" +#define PX4_GIT_BRANCH_NAME "{git_branch_name}" #define PX4_GIT_OEM_VERSION_STR "{oem_tag}" -""".format(git_tag=git_tag, - git_version=git_version, - git_version_short=git_version_short, - git_branch_name=git_branch_name, - oem_tag=oem_tag) + +#define PX4_GIT_TAG_OR_BRANCH_NAME "{tag_or_branch}" // special variable: git tag, release or master branch +""" # ECL @@ -115,11 +128,10 @@ if (os.path.exists('src/lib/ecl/.git')): cwd='src/lib/ecl', stderr=subprocess.STDOUT).decode('utf-8').strip() ecl_git_version_short = ecl_git_version[0:16] - header += """ + header += f""" #define ECL_LIB_GIT_VERSION_STR "{ecl_git_version}" #define ECL_LIB_GIT_VERSION_BINARY 0x{ecl_git_version_short} -""".format(ecl_git_version=ecl_git_version, - ecl_git_version_short=ecl_git_version_short) +""" # Mavlink @@ -128,11 +140,10 @@ if (os.path.exists('mavlink/include/mavlink/v2.0/.git')): cwd='mavlink/include/mavlink/v2.0', stderr=subprocess.STDOUT).decode('utf-8').strip() mavlink_git_version_short = mavlink_git_version[0:16] - header += """ + header += f""" #define MAVLINK_LIB_GIT_VERSION_STR "{mavlink_git_version}" #define MAVLINK_LIB_GIT_VERSION_BINARY 0x{mavlink_git_version_short} -""".format(mavlink_git_version=mavlink_git_version, - mavlink_git_version_short=mavlink_git_version_short) +""" # NuttX @@ -144,13 +155,11 @@ if (os.path.exists('platforms/nuttx/NuttX/nuttx/.git')): cwd='platforms/nuttx/NuttX/nuttx', stderr=subprocess.STDOUT).decode('utf-8').strip() nuttx_git_version_short = nuttx_git_version[0:16] - header += """ + header += f""" #define NUTTX_GIT_VERSION_STR "{nuttx_git_version}" #define NUTTX_GIT_VERSION_BINARY 0x{nuttx_git_version_short} #define NUTTX_GIT_TAG_STR "{nuttx_git_tag}" -""".format(nuttx_git_version=nuttx_git_version, - nuttx_git_version_short=nuttx_git_version_short, - nuttx_git_tag=nuttx_git_tag) +""" if old_header != header: