Clean Up Unit Processor Code

Author: David Cooper

fitparse/base.py

@@ -11,22 +11,14 @@
     DataMessage, FieldData, FieldDefinition, DefinitionMessage, MessageHeader,
     BASE_TYPES, BASE_TYPE_BYTE
 )
-from fitparse.utils import calc_crc
+from fitparse.utils import calc_crc, scrub_method_name
 
 
 class FitParseError(Exception):
     pass
 
 
 class FitFile(object):
-    # TODO: unit test to make sure that all units in profile.py convert to
-    #       sane function names after applying replacements (and there are no
-    #       no regressions)
-    UNIT_NAME_TO_FUNC_REPLACEMENTS = (
-        ('/', 'per'),
-        ('%', 'percent'),
-    )
-
     def __init__(self, fileish, check_crc=True, data_processor=None):
         if hasattr(fileish, 'read'):
             self._file = fileish
@@ -287,9 +279,10 @@ def _parse_data_message(self, header):
                 # Resolve component fields
                 if field.components:
                     for component in field.components:
-                        # Render it's raw value
+                        # Render its raw value
                         cmp_raw_value = component.render(raw_value)
 
+                        # Apply accumulated value
                         if component.accumulate:
                             accumulator = self._accumulators[def_mesg.mesg_num]
                             cmp_raw_value = self._apply_compressed_accumulation(
@@ -303,6 +296,7 @@ def _parse_data_message(self, header):
 
                         # Extract the component's dynamic field from def_mesg
                         cmp_field = def_mesg.mesg_type.fields[component.def_num]
+
                         # Resolve a possible subfield
                         cmp_field, cmp_parent_field = self._resolve_subfield(cmp_field, def_mesg, raw_values)
                         cmp_value = cmp_field.render(cmp_raw_value)
@@ -356,32 +350,28 @@ def _parse_data_message(self, header):
         # Apply data processors
         for field_data in field_datas:
             # Apply type name processor
-            type_processor = getattr(self._processor, 'process_type_%s' % field_data.type.name, None)
+            process_method_name = scrub_method_name('process_type_%s' % field_data.type.name)
+            type_processor = getattr(self._processor, process_method_name, None)
             if type_processor:
                 type_processor(field_data)
 
             # Apply field name processor
-            field_processor = getattr(self._processor, 'process_field_%s' % field_data.name, None)
+            process_method_name = scrub_method_name('process_field_%s' % field_data.name)
+            field_processor = getattr(self._processor, process_method_name, None)
             if field_processor:
                 field_processor(field_data)
 
             # Apply units name processor
             if field_data.units:
-                process_func_name = 'process_units_%s' % field_data.units
-                # Do unit name replacements padded with spaces
-                for replace_from, replace_to in self.UNIT_NAME_TO_FUNC_REPLACEMENTS:
-                    process_func_name = process_func_name.replace(
-                        replace_from, ' %s ' % replace_to,
-                    )
-                # Then strip and convert spaces to underscores
-                process_func_name = process_func_name.strip().replace(' ', '_')
-                units_processor = getattr(self._processor, process_func_name, None)
+                process_method_name = scrub_method_name('process_units_%s' % field_data.units, convert_units=True)
+                units_processor = getattr(self._processor, process_method_name, None)
                 if units_processor:
                     units_processor(field_data)
 
         data_message = DataMessage(header=header, def_mesg=def_mesg, fields=field_datas)
 
-        mesg_processor = getattr(self._processor, 'process_message_%s' % def_mesg.name, None)
+        process_method_name = scrub_method_name('process_message_%s' % def_mesg.name)
+        mesg_processor = getattr(self._processor, process_method_name, None)
         if mesg_processor:
             mesg_processor(data_message)
 

fitparse/profile.py

@@ -1,7 +1,7 @@
 
 ################# BEGIN AUTOMATICALLY GENERATED FIT PROFILE ##################
 ########################### DO NOT EDIT THIS FILE ############################
-####### EXPORTED PROFILE FROM SDK VERSION 13.0 AT 2014-11-09 15:34:10 ########
+####### EXPORTED PROFILE FROM SDK VERSION 13.0 AT 2014-11-09 16:44:30 ########
 ########### PARSED 64 TYPES (813 VALUES), 41 MESSAGES (519 FIELDS) ###########
 
 from fitparse.records import (
@@ -4094,13 +4094,13 @@
                 name='distance_16',
                 type=BASE_TYPES[0x84],  # uint16
                 def_num=8,
-                units='100 * m',
+                units='100*m',
             ),
             9: Field(
                 name='cycles_16',
                 type=BASE_TYPES[0x84],  # uint16
                 def_num=9,
-                units='2 * cycles (steps)',
+                units='2*cycles or steps',
             ),
             10: Field(
                 name='active_time_16',
@@ -4238,7 +4238,7 @@
                 name='resting_metabolic_rate',
                 type=BASE_TYPES[0x84],  # uint16
                 def_num=5,
-                units='kcal / day',
+                units='kcal/day',
             ),
             253: FIELD_TYPE_TIMESTAMP,
         },

fitparse/utils.py

@@ -1,3 +1,6 @@
+import re
+
+
 CRC_TABLE = (
     0x0000, 0xCC01, 0xD801, 0x1400, 0xF001, 0x3C00, 0x2800, 0xE401,
     0xA001, 0x6C00, 0x7800, 0xB401, 0x5000, 0x9C01, 0x8801, 0x4400,
@@ -16,3 +19,19 @@ def calc_crc(bytes, crc=0):
         crc = (crc >> 4) & 0x0FFF
         crc = crc ^ tmp ^ CRC_TABLE[(byte_char >> 4) & 0xF]
     return crc
+
+
+METHOD_NAME_SCRUBBER = re.compile(r'\W|^(?=\d)')
+UNIT_NAME_TO_FUNC_REPLACEMENTS = (
+    ('/', ' per '),
+    ('%', 'percent'),
+    ('*', ' times '),
+)
+
+def scrub_method_name(method_name, convert_units=False):
+    if convert_units:
+        for replace_from, replace_to in UNIT_NAME_TO_FUNC_REPLACEMENTS:
+            method_name = method_name.replace(
+                replace_from, '%s' % replace_to,
+            )
+    return METHOD_NAME_SCRUBBER.sub('_', method_name)

scripts/generate_profile.py

@@ -264,8 +264,9 @@ def fix_scale(data):
 
 def fix_units(data):
     if isinstance(data, basestring):
-        if data == 'kcal / min':
-            data = 'kcal/min'
+        data = data.replace(' / ', '/')
+        data = data.replace(' * ', '*')
+        data = data.replace('(steps)', 'or steps')
         data = data.strip()
     return data
 

scripts/unit_tool.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+# Tool for verifying sanity of units in Profile.xls / fitparse/profile.py
+
+import os
+import sys
+
+import xlrd  # Dev requirement for parsing Excel spreadsheet
+
+from fitparse.profile import MESSAGE_TYPES
+from fitparse.utils import scrub_method_name
+
+
+def do_profile_xls():
+    workbook = xlrd.open_workbook(sys.argv[1])
+    sheet = workbook.sheet_by_name('Messages')
+
+    all_unit_values = []
+    for unit_value in sheet.col_values(8):  # Extract unit column values
+        unit_value = unit_value.strip()
+        if unit_value:
+            # Deal with comma separated components
+            unit_values = [v.strip() for v in unit_value.split(',')]
+            all_unit_values.extend(unit_values)
+
+    print 'In Profile.xls:'
+    for unit_value in sorted(set(all_unit_values)):
+        print ' * %s' % unit_value
+
+
+def do_fitparse_profile():
+    unit_values = []
+    for message_type in MESSAGE_TYPES.values():
+        for field in message_type.fields.values():
+            unit_values.append(field.units)
+            if field.components:
+                for component in field.components:
+                    unit_values.append(component.units)
+            if field.subfields:
+                for subfield in field.subfields:
+                    unit_values.append(subfield.units)
+                    if subfield.components:
+                        for component in subfield.components:
+                            unit_values.append(component.units)
+
+    unit_values = filter(None, unit_values)
+
+    print 'In fitparse/profile.py:'
+    for unit_value in sorted(set(unit_values)):
+        print ' * %s [%s]' % (
+            unit_value,
+            scrub_method_name('process_units_%s' % unit_value, convert_units=True),
+        )
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print "Usage: %s Profile.xls" % os.path.basename(__file__)
+        sys.exit(0)
+
+    do_profile_xls()
+    print
+    do_fitparse_profile()
+
+

tests/test.py

@@ -219,6 +219,11 @@ def test_parsing_edge_500_fit_file(self):
         except StopIteration:
             pass
 
+    # TODO:
+    #  * Test Processors:
+    #    - process_type_<>, process_field_<>, process_units_<>, process_message_<>
+    #  * Test subfield components (have to create a sample fit file)
+
 
 if __name__ == '__main__':
     unittest.main()