refactor Tools/serial: move non-serial param generation into separate script

Tools/module_config/generate_params.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+""" Script to params from module.yaml config file(s)
+    Note: serial params are handled in Tools/serial/generate_config.py
+"""
+
+import argparse
+import os
+import sys
+
+
+try:
+    import yaml
+except ImportError as e:
+    print("Failed to import yaml: " + str(e))
+    print("")
+    print("You may need to install it using:")
+    print("    pip3 install --user pyyaml")
+    print("")
+    sys.exit(1)
+
+parser = argparse.ArgumentParser(description='Generate params from module.yaml file(s)')
+
+parser.add_argument('--config-files', type=str, nargs='*', default=[],
+                    help='YAML module config file(s)')
+parser.add_argument('--params-file', type=str, action='store',
+                    help='Parameter output file')
+parser.add_argument('--ethernet', action='store_true',
+                    help='Ethernet support')
+parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
+                    help='Verbose Output')
+
+args = parser.parse_args()
+
+verbose = args.verbose
+params_output_file = args.params_file
+ethernet_supported = args.ethernet
+
+def parse_yaml_parameters_config(yaml_config, ethernet_supported):
+    """ parse the parameters section from the yaml config file """
+    if 'parameters' not in yaml_config:
+        return ''
+    parameters_section_list = yaml_config['parameters']
+    for parameters_section in parameters_section_list:
+        if 'definitions' not in parameters_section:
+            return ''
+        definitions = parameters_section['definitions']
+        ret = ''
+        param_group = parameters_section.get('group', None)
+        for param_name in definitions:
+            param = definitions[param_name]
+            if param.get('requires_ethernet', False) and not ethernet_supported:
+                continue
+            num_instances = param.get('num_instances', 1)
+            instance_start = param.get('instance_start', 0) # offset
+
+            # get the type and extract all tags
+            tags = '@group {:}'.format(param_group)
+            if param['type'] == 'enum':
+                param_type = 'INT32'
+                for key in param['values']:
+                    tags += '\n * @value {:} {:}'.format(key, param['values'][key])
+            elif param['type'] == 'boolean':
+                param_type = 'INT32'
+                tags += '\n * @boolean'
+            elif param['type'] == 'int32':
+                param_type = 'INT32'
+            elif param['type'] == 'float':
+                param_type = 'FLOAT'
+            else:
+                raise Exception("unknown param type {:}".format(param['type']))
+
+            for tag in ['decimal', 'increment', 'category', 'volatile', 'bit',
+                        'min', 'max', 'unit', 'reboot_required']:
+                if tag in param:
+                    tags += '\n * @{:} {:}'.format(tag, param[tag])
+
+            for i in range(num_instances):
+                # default value
+                default_value = 0
+                if 'default' in param:
+                    # default can be a list of num_instances or a single value
+                    if type(param['default']) == list:
+                        assert len(param['default']) == num_instances
+                        default_value = param['default'][i]
+                    else:
+                        default_value = param['default']
+
+                if type(default_value) == bool:
+                    default_value = int(default_value)
+
+                # output the existing C-style format
+                ret += '''
+/**
+ * {short_descr}
+ *
+ * {long_descr}
+ *
+ * {tags}
+ */
+PARAM_DEFINE_{param_type}({name}, {default_value});
+'''.format(short_descr=param['description']['short'].replace("\n", "\n * "),
+           long_descr=param['description']['long'].replace("\n", "\n * "),
+           tags=tags,
+           param_type=param_type,
+           name=param_name,
+           default_value=default_value,
+          ).replace('${i}', str(i+instance_start))
+    return ret
+
+all_params = ""
+
+for yaml_file in args.config_files:
+    with open(yaml_file, 'r') as stream:
+        try:
+            yaml_config = yaml.load(stream, Loader=yaml.Loader)
+
+            all_params += parse_yaml_parameters_config(yaml_config, ethernet_supported)
+
+        except yaml.YAMLError as exc:
+            print(exc)
+            raise
+
+if verbose: print("Generating {:}".format(params_output_file))
+with open(params_output_file, 'w') as fid:
+    fid.write(all_params)
+

Tools/serial/generate_config.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 """ Script to generate Serial (UART) parameters and the ROMFS startup script """
 
-from __future__ import print_function
-
 import argparse
 import os
 import sys
@@ -195,7 +193,6 @@ if rc_serial_output_dir is None and serial_params_output_file is None:
 # parse the YAML files
 serial_commands = []
 ethernet_configuration = []
-additional_params = ""
 
 if ethernet_supported:
     ethernet_configuration.append({
@@ -216,88 +213,12 @@ def parse_yaml_serial_config(yaml_config):
         ret.append(serial_config)
     return ret
 
-def parse_yaml_parameters_config(yaml_config, ethernet_supported):
-    """ parse the parameters section from the yaml config file """
-    if 'parameters' not in yaml_config:
-        return ''
-    parameters_section_list = yaml_config['parameters']
-    for parameters_section in parameters_section_list:
-        if 'definitions' not in parameters_section:
-            return ''
-        definitions = parameters_section['definitions']
-        ret = ''
-        param_group = parameters_section.get('group', None)
-        for param_name in definitions:
-            param = definitions[param_name]
-            if param.get('requires_ethernet', False) and not ethernet_supported:
-                continue
-            num_instances = param.get('num_instances', 1)
-            instance_start = param.get('instance_start', 0) # offset
-
-            # get the type and extract all tags
-            tags = '@group {:}'.format(param_group)
-            if param['type'] == 'enum':
-                param_type = 'INT32'
-                for key in param['values']:
-                    tags += '\n * @value {:} {:}'.format(key, param['values'][key])
-            elif param['type'] == 'boolean':
-                param_type = 'INT32'
-                tags += '\n * @boolean'
-            elif param['type'] == 'int32':
-                param_type = 'INT32'
-            elif param['type'] == 'float':
-                param_type = 'FLOAT'
-            else:
-                raise Exception("unknown param type {:}".format(param['type']))
-
-            for tag in ['decimal', 'increment', 'category', 'volatile', 'bit',
-                        'min', 'max', 'unit', 'reboot_required']:
-                if tag in param:
-                    tags += '\n * @{:} {:}'.format(tag, param[tag])
-
-            for i in range(num_instances):
-                # default value
-                default_value = 0
-                if 'default' in param:
-                    # default can be a list of num_instances or a single value
-                    if type(param['default']) == list:
-                        assert len(param['default']) == num_instances
-                        default_value = param['default'][i]
-                    else:
-                        default_value = param['default']
-
-                if type(default_value) == bool:
-                    default_value = int(default_value)
-
-                # output the existing C-style format
-                ret += '''
-/**
- * {short_descr}
- *
- * {long_descr}
- *
- * {tags}
- */
-PARAM_DEFINE_{param_type}({name}, {default_value});
-'''.format(short_descr=param['description']['short'].replace("\n", "\n * "),
-           long_descr=param['description']['long'].replace("\n", "\n * "),
-           tags=tags,
-           param_type=param_type,
-           name=param_name,
-           default_value=default_value,
-          ).replace('${i}', str(i+instance_start))
-    return ret
-
 for yaml_file in args.config_files:
     with open(yaml_file, 'r') as stream:
         try:
             yaml_config = yaml.load(stream, Loader=yaml.Loader)
             serial_commands.extend(parse_yaml_serial_config(yaml_config))
 
-            # TODO: additional params should be parsed in a separate script
-            additional_params += parse_yaml_parameters_config(
-                yaml_config, ethernet_supported)
-
         except yaml.YAMLError as exc:
             print(exc)
             raise
@@ -404,6 +325,5 @@ if serial_params_output_file is not None:
     with open(serial_params_output_file, 'w') as fid:
         fid.write(template.render(serial_devices=serial_devices,
             ethernet_configuration=ethernet_configuration,
-            commands=commands, serial_ports=serial_ports,
-            additional_definitions=additional_params))
+            commands=commands, serial_ports=serial_ports))
 

Tools/serial/serial_params.c.jinja

@@ -71,5 +71,3 @@ PARAM_DEFINE_INT32({{ command.port_param_name }}, {{ command.default_port }});
 {% endfor -%}
 {% endif %}
 
-{{ additional_definitions }}
-

src/lib/parameters/CMakeLists.txt

@@ -71,23 +71,29 @@ list(REMOVE_DUPLICATES module_list)
 
 set(generated_params_dir ${PX4_BINARY_DIR}/generated_params)
 set(generated_serial_params_file ${generated_params_dir}/serial_params.c)
+set(generated_module_params_file ${generated_params_dir}/module_params.c)
 file(GLOB jinja_templates ${PX4_SOURCE_DIR}/Tools/serial/*.jinja)
 if (px4_constrained_flash_build)
-	set(added_arguments --constrained-flash)
+	set(constrained_flash_arg --constrained-flash)
 endif()
 if(PX4_ETHERNET)
-	set(added_arguments ${added_arguments} --ethernet)
+	set(added_arguments --ethernet)
 endif()
-add_custom_command(OUTPUT ${generated_serial_params_file}
+add_custom_command(OUTPUT ${generated_serial_params_file} ${generated_module_params_file}
 	COMMAND ${CMAKE_COMMAND} -E make_directory ${generated_params_dir}
 	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/serial/generate_config.py
 		--params-file ${generated_serial_params_file}
-		--serial-ports ${board_serial_ports} ${added_arguments}
+		--serial-ports ${board_serial_ports} ${added_arguments} ${constrained_flash_arg}
+		--config-files ${module_config_files} #--verbose
+	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/module_config/generate_params.py
+		--params-file ${generated_module_params_file}
+		${added_arguments}
 		--config-files ${module_config_files} #--verbose
 	DEPENDS
 		${module_config_files}
 		${jinja_templates}
 		${PX4_SOURCE_DIR}/Tools/serial/generate_config.py
+		${PX4_SOURCE_DIR}/Tools/module_config/generate_params.py
 	COMMENT "Generating serial_params.c"
 )
 
@@ -107,6 +113,7 @@ add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json
 	DEPENDS
 		${param_src_files}
 		${generated_serial_params_file}
+		${generated_module_params_file}
 		parameters_injected.xml
 		px4params/srcparser.py
 		px4params/srcscanner.py