1#!/usr/bin/env python3
2# vim: set syntax=python ts=4 :
3#
4# Copyright (c) 2018 Intel Corporation
5# SPDX-License-Identifier: Apache-2.0
6
7import subprocess
8import sys
9import os
10import re
11import typing
12import logging
13from twisterlib.error import TwisterRuntimeError
14
15logger = logging.getLogger('twister')
16logger.setLevel(logging.DEBUG)
17
18class SizeCalculator:
19    alloc_sections = [
20        "bss",
21        "noinit",
22        "app_bss",
23        "app_noinit",
24        "ccm_bss",
25        "ccm_noinit"
26    ]
27
28    rw_sections = [
29        "datas",
30        "initlevel",
31        "exceptions",
32        "_static_thread_data_area",
33        "k_timer_area",
34        "k_mem_slab_area",
35        "sw_isr_table",
36        "k_sem_area",
37        "k_mutex_area",
38        "app_shmem_regions",
39        "_k_fifo_area",
40        "_k_lifo_area",
41        "k_stack_area",
42        "k_msgq_area",
43        "k_mbox_area",
44        "k_pipe_area",
45        "net_if_area",
46        "net_if_dev_area",
47        "net_l2_area",
48        "net_l2_data",
49        "k_queue_area",
50        "_net_buf_pool_area",
51        "app_datas",
52        "kobject_data",
53        "mmu_tables",
54        "app_pad",
55        "priv_stacks",
56        "ccm_data",
57        "usb_descriptor",
58        "usb_data", "usb_bos_desc",
59        "uart_mux",
60        'log_backends_sections',
61        'log_dynamic_sections',
62        'log_const_sections',
63        "app_smem",
64        'shell_root_cmds_sections',
65        'log_const_sections',
66        "priv_stacks_noinit",
67        "_GCOV_BSS_SECTION_NAME",
68        "gcov",
69        "nocache",
70        "devices",
71        "k_heap_area",
72    ]
73
74    # These get copied into RAM only on non-XIP
75    ro_sections = [
76        "rom_start",
77        "text",
78        "ctors",
79        "init_array",
80        "reset",
81        "k_object_assignment_area",
82        "rodata",
83        "net_l2",
84        "vector",
85        "sw_isr_table",
86        "settings_handler_static_area",
87        "bt_l2cap_fixed_chan_area",
88        "bt_l2cap_br_fixed_chan_area",
89        "bt_gatt_service_static_area",
90        "vectors",
91        "net_socket_register_area",
92        "net_ppp_proto",
93        "shell_area",
94        "tracing_backend_area",
95        "ppp_protocol_handler_area",
96    ]
97
98    # Variable below is stored for calculating size using build.log
99    USEFUL_LINES_AMOUNT = 4
100
101    def __init__(self, elf_filename: str,\
102        extra_sections: typing.List[str],\
103        buildlog_filepath: str = '',\
104        generate_warning: bool = True):
105        """Constructor
106
107        @param elf_filename (str) Path to the output binary
108            parsed by objdump to determine section sizes.
109        @param extra_sections (list[str]) List of extra,
110            unexpected sections, which Twister should not
111            report as error and not include in the
112            size calculation.
113        @param buildlog_filepath (str, default: '') Path to the
114            output build.log
115        @param generate_warning (bool, default: True) Twister should
116            (or not) warning about missing the information about
117            footprint.
118        """
119        self.elf_filename = elf_filename
120        self.buildlog_filename = buildlog_filepath
121        self.sections = []
122        self.used_rom = 0
123        self.used_ram = 0
124        self.available_ram = 0
125        self.available_rom = 0
126        self.extra_sections = extra_sections
127        self.is_xip = True
128        self.generate_warning = generate_warning
129
130        self._calculate_sizes()
131
132    def size_report(self):
133        print(self.elf_filename)
134        print("SECTION NAME             VMA        LMA     SIZE  HEX SZ TYPE")
135        for v in self.sections:
136            print("%-17s 0x%08x 0x%08x %8d 0x%05x %-7s" %
137                        (v["name"], v["virt_addr"], v["load_addr"], v["size"], v["size"],
138                        v["type"]))
139
140        print("Totals: %d bytes (ROM), %d bytes (RAM)" %
141                    (self.used_rom, self.used_ram))
142        print("")
143
144    def get_used_ram(self):
145        """Get the amount of RAM the application will use up on the device
146
147        @return amount of RAM, in bytes
148        """
149        return self.used_ram
150
151    def get_used_rom(self):
152        """Get the size of the data that this application uses on device's flash
153
154        @return amount of ROM, in bytes
155        """
156        return self.used_rom
157
158    def unrecognized_sections(self):
159        """Get a list of sections inside the binary that weren't recognized
160
161        @return list of unrecognized section names
162        """
163        slist = []
164        for v in self.sections:
165            if not v["recognized"]:
166                slist.append(v["name"])
167        return slist
168
169    def get_available_ram(self) -> int:
170        """Get the total available RAM.
171
172        @return total available RAM, in bytes (int)
173        """
174        return self.available_ram
175
176    def get_available_rom(self) -> int:
177        """Get the total available ROM.
178
179        @return total available ROM, in bytes (int)
180        """
181        return self.available_rom
182
183    def _calculate_sizes(self):
184        """ELF file is analyzed, even if the option to read memory
185        footprint from the build.log file is set.
186        This is to detect potential problems contained in
187        unrecognized sections of the file.
188        """
189        self._analyze_elf_file()
190        if self.buildlog_filename.endswith("build.log"):
191            self._get_footprint_from_buildlog()
192
193    def _check_elf_file(self) -> None:
194        # Make sure this is an ELF binary
195        with open(self.elf_filename, "rb") as f:
196            magic = f.read(4)
197
198        try:
199            if magic != b'\x7fELF':
200                raise TwisterRuntimeError("%s is not an ELF binary" % self.elf_filename)
201        except Exception as e:
202            print(str(e))
203            sys.exit(2)
204
205    def _check_is_xip(self) -> None:
206        # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
207        # GREP can not be used as it returns an error if the symbol is not
208        # found.
209        is_xip_command = "nm " + self.elf_filename + \
210                         " | awk '/CONFIG_XIP/ { print $3 }'"
211        is_xip_output = subprocess.check_output(
212            is_xip_command, shell=True, stderr=subprocess.STDOUT).decode(
213            "utf-8").strip()
214        try:
215            if is_xip_output.endswith("no symbols"):
216                raise TwisterRuntimeError("%s has no symbol information" % self.elf_filename)
217        except Exception as e:
218            print(str(e))
219            sys.exit(2)
220
221        self.is_xip = len(is_xip_output) != 0
222
223    def _get_info_elf_sections(self) -> None:
224        """Calculate RAM and ROM usage and information about issues by section"""
225        objdump_command = "objdump -h " + self.elf_filename
226        objdump_output = subprocess.check_output(
227            objdump_command, shell=True).decode("utf-8").splitlines()
228
229        for line in objdump_output:
230            words = line.split()
231
232            if not words:  # Skip lines that are too short
233                continue
234
235            index = words[0]
236            if not index[0].isdigit():  # Skip lines that do not start
237                continue  # with a digit
238
239            name = words[1]  # Skip lines with section names
240            if name[0] == '.':  # starting with '.'
241                continue
242
243            # TODO this doesn't actually reflect the size in flash or RAM as
244            # it doesn't include linker-imposed padding between sections.
245            # It is close though.
246            size = int(words[2], 16)
247            if size == 0:
248                continue
249
250            load_addr = int(words[4], 16)
251            virt_addr = int(words[3], 16)
252
253            # Add section to memory use totals (for both non-XIP and XIP scenarios)
254            # Unrecognized section names are not included in the calculations.
255            recognized = True
256
257            # If build.log file exists, check errors (unrecognized sections
258            # in ELF file).
259            if self.buildlog_filename:
260                if name in SizeCalculator.alloc_sections or \
261                   name in SizeCalculator.rw_sections or \
262                   name in SizeCalculator.ro_sections:
263                    continue
264                else:
265                    stype = "unknown"
266                    if name not in self.extra_sections:
267                        recognized = False
268            else:
269                if name in SizeCalculator.alloc_sections:
270                    self.used_ram += size
271                    stype = "alloc"
272                elif name in SizeCalculator.rw_sections:
273                    self.used_ram += size
274                    self.used_rom += size
275                    stype = "rw"
276                elif name in SizeCalculator.ro_sections:
277                    self.used_rom += size
278                    if not self.is_xip:
279                        self.used_ram += size
280                    stype = "ro"
281                else:
282                    stype = "unknown"
283                    if name not in self.extra_sections:
284                        recognized = False
285
286            self.sections.append({"name": name, "load_addr": load_addr,
287                                  "size": size, "virt_addr": virt_addr,
288                                  "type": stype, "recognized": recognized})
289
290    def _analyze_elf_file(self) -> None:
291        self._check_elf_file()
292        self._check_is_xip()
293        self._get_info_elf_sections()
294
295    def _get_buildlog_file_content(self) -> typing.List[str]:
296        """Get content of the build.log file.
297
298        @return Content of the build.log file (list[str])
299        """
300        if os.path.exists(path=self.buildlog_filename):
301            with open(file=self.buildlog_filename, mode='r') as file:
302                file_content = file.readlines()
303        else:
304            if self.generate_warning:
305                logger.error(msg=f"Incorrect path to build.log file to analyze footprints. Please check the path {self.buildlog_filename}.")
306            file_content = []
307        return file_content
308
309    def _find_offset_of_last_pattern_occurrence(self, file_content: typing.List[str]) -> int:
310        """Find the offset from which the information about the memory footprint is read.
311
312        @param file_content (list[str]) Content of build.log.
313        @return Offset with information about the memory footprint (int)
314        """
315        result = -1
316        if len(file_content) == 0:
317            logger.warning("Build.log file is empty.")
318        else:
319            # Pattern to first line with information about memory footprint
320            PATTERN_SEARCHED_LINE = "Memory region"
321            # Check the file in reverse order.
322            for idx, line in enumerate(reversed(file_content)):
323                # Condition is fulfilled if the pattern matches with the start of the line.
324                if re.match(pattern=PATTERN_SEARCHED_LINE, string=line):
325                    result = idx + 1
326                    break
327        # If the file does not contain information about memory footprint, the warning is raised.
328        if result == -1:
329            logger.warning(msg=f"Information about memory footprint for this test configuration is not found. Please check file {self.buildlog_filename}.")
330        return result
331
332    def _get_lines_with_footprint(self, start_offset: int, file_content: typing.List[str]) -> typing.List[str]:
333        """Get lines from the file with a memory footprint.
334
335        @param start_offset (int) Offset with the first line of the information about memory footprint.
336        @param file_content (list[str]) Content of the build.log file.
337        @return Lines with information about memory footprint (list[str])
338        """
339        if len(file_content) == 0:
340            result = []
341        else:
342            if start_offset > len(file_content) or start_offset <= 0:
343                info_line_idx_start = len(file_content) - 1
344            else:
345                info_line_idx_start = len(file_content) - start_offset
346
347            info_line_idx_stop = info_line_idx_start + self.USEFUL_LINES_AMOUNT
348            if info_line_idx_stop > len(file_content):
349                info_line_idx_stop = len(file_content)
350
351            result = file_content[info_line_idx_start:info_line_idx_stop]
352        return result
353
354    def _clear_whitespaces_from_lines(self, text_lines: typing.List[str]) -> typing.List[str]:
355        """Clear text lines from whitespaces.
356
357        @param text_lines (list[str]) Lines with useful information.
358        @return  Cleared text lines with information (list[str])
359        """
360        return [line.strip("\n").rstrip("%") for line in text_lines] if text_lines else []
361
362    def _divide_text_lines_into_columns(self, text_lines: typing.List[str]) -> typing.List[typing.List[str]]:
363        """Divide lines of text into columns.
364
365        @param lines (list[list[str]]) Lines with information about memory footprint.
366        @return Lines divided into columns (list[list[str]])
367        """
368        if text_lines:
369            result = []
370            PATTERN_SPLIT_COLUMNS = "  +"
371            for line in text_lines:
372                line = [column.rstrip(":") for column in re.split(pattern=PATTERN_SPLIT_COLUMNS, string=line)]
373                result.append(list(filter(None, line)))
374        else:
375            result = [[]]
376
377        return result
378
379    def _unify_prefixes_on_all_values(self, data_lines: typing.List[typing.List[str]]) -> typing.List[typing.List[str]]:
380        """Convert all values in the table to unified order of magnitude.
381
382        @param data_lines (list[list[str]]) Lines with information about memory footprint.
383        @return Lines with unified values (list[list[str]])
384        """
385        if len(data_lines) != self.USEFUL_LINES_AMOUNT:
386            data_lines = [[]]
387            if self.generate_warning:
388                logger.warning(msg=f"Incomplete information about memory footprint. Please check file {self.buildlog_filename}")
389        else:
390            for idx, line in enumerate(data_lines):
391                # Line with description of the columns
392                if idx == 0:
393                    continue
394                line_to_replace = list(map(self._binary_prefix_converter, line))
395                data_lines[idx] = line_to_replace
396
397        return data_lines
398
399    def _binary_prefix_converter(self, value: str) -> str:
400        """Convert specific value to particular prefix.
401
402        @param value (str) Value to convert.
403        @return Converted value to output prefix (str)
404        """
405        PATTERN_VALUE = r"([0-9]?\s.?B\Z)"
406
407        if not re.search(pattern=PATTERN_VALUE, string=value):
408            converted_value = value.rstrip()
409        else:
410            PREFIX_POWER = {'B': 0, 'KB': 10, 'MB': 20, 'GB': 30}
411            DEFAULT_DATA_PREFIX = 'B'
412
413            data_parts = value.split()
414            numeric_value = int(data_parts[0])
415            unit = data_parts[1]
416            shift = PREFIX_POWER.get(unit, 0) - PREFIX_POWER.get(DEFAULT_DATA_PREFIX, 0)
417            unit_predictor = pow(2, shift)
418            converted_value = str(numeric_value * unit_predictor)
419        return converted_value
420
421    def _create_data_table(self) -> typing.List[typing.List[str]]:
422        """Create table with information about memory footprint.
423
424        @return Table with information about memory usage (list[list[str]])
425        """
426        file_content = self._get_buildlog_file_content()
427        data_line_start_idx = self._find_offset_of_last_pattern_occurrence(file_content=file_content)
428
429        if data_line_start_idx < 0:
430            data_from_content = [[]]
431        else:
432            # Clean lines and separate information to columns
433            information_lines = self._get_lines_with_footprint(start_offset=data_line_start_idx, file_content=file_content)
434            information_lines = self._clear_whitespaces_from_lines(text_lines=information_lines)
435            data_from_content = self._divide_text_lines_into_columns(text_lines=information_lines)
436            data_from_content = self._unify_prefixes_on_all_values(data_lines=data_from_content)
437
438        return data_from_content
439
440    def _get_footprint_from_buildlog(self) -> None:
441        """Get memory footprint from build.log"""
442        data_from_file = self._create_data_table()
443
444        if data_from_file == [[]] or not data_from_file:
445            self.used_ram = 0
446            self.used_rom = 0
447            self.available_ram = 0
448            self.available_rom = 0
449            if self.generate_warning:
450                logger.warning(msg=f"Missing information about memory footprint. Check file {self.buildlog_filename}.")
451        else:
452            ROW_RAM_IDX = 2
453            ROW_ROM_IDX = 1
454            COLUMN_USED_SIZE_IDX = 1
455            COLUMN_AVAILABLE_SIZE_IDX = 2
456
457            self.used_ram = int(data_from_file[ROW_RAM_IDX][COLUMN_USED_SIZE_IDX])
458            self.used_rom = int(data_from_file[ROW_ROM_IDX][COLUMN_USED_SIZE_IDX])
459            self.available_ram = int(data_from_file[ROW_RAM_IDX][COLUMN_AVAILABLE_SIZE_IDX])
460            self.available_rom = int(data_from_file[ROW_ROM_IDX][COLUMN_AVAILABLE_SIZE_IDX])
461