1#!/usr/bin/env python3
2# vim: set syntax=python ts=4 :
3#
4# Copyright (c) 2018 Intel Corporation
5# SPDX-License-Identifier: Apache-2.0
6
7import logging
8import os
9import re
10import subprocess
11import sys
12
13from twisterlib.error import TwisterRuntimeError
14
15logger = logging.getLogger('twister')
16logger.setLevel(logging.DEBUG)
17
18class SizeCalculator:
19    alloc_sections = [
20        "bss",
21        "noinit",
22        "app_bss",
23        "app_noinit",
24        "ccm_bss",
25        "ccm_noinit"
26    ]
27
28    rw_sections = [
29        "datas",
30        "initlevel",
31        "exceptions",
32        "_static_thread_data_area",
33        "k_timer_area",
34        "k_mem_slab_area",
35        "sw_isr_table",
36        "k_sem_area",
37        "k_mutex_area",
38        "app_shmem_regions",
39        "_k_fifo_area",
40        "_k_lifo_area",
41        "k_stack_area",
42        "k_msgq_area",
43        "k_mbox_area",
44        "k_pipe_area",
45        "net_if_area",
46        "net_if_dev_area",
47        "net_l2_area",
48        "net_l2_data",
49        "k_queue_area",
50        "_net_buf_pool_area",
51        "app_datas",
52        "kobject_data",
53        "mmu_tables",
54        "app_pad",
55        "priv_stacks",
56        "ccm_data",
57        "usb_descriptor",
58        "usb_data", "usb_bos_desc",
59        'log_backends_sections',
60        'log_dynamic_sections',
61        'log_const_sections',
62        "app_smem",
63        'shell_root_cmds_sections',
64        'log_const_sections',
65        "priv_stacks_noinit",
66        "_GCOV_BSS_SECTION_NAME",
67        "gcov",
68        "nocache",
69        "devices",
70        "k_heap_area",
71    ]
72
73    # These get copied into RAM only on non-XIP
74    ro_sections = [
75        "rom_start",
76        "text",
77        "ctors",
78        "init_array",
79        "reset",
80        "k_object_assignment_area",
81        "rodata",
82        "net_l2",
83        "vector",
84        "sw_isr_table",
85        "settings_handler_static_area",
86        "bt_l2cap_fixed_chan_area",
87        "bt_l2cap_br_fixed_chan_area",
88        "bt_gatt_service_static_area",
89        "vectors",
90        "net_socket_register_area",
91        "net_ppp_proto",
92        "shell_area",
93        "tracing_backend_area",
94        "ppp_protocol_handler_area",
95    ]
96
97    # Variable below is stored for calculating size using build.log
98    USEFUL_LINES_AMOUNT = 4
99
100    def __init__(self, elf_filename: str,\
101        extra_sections: list[str],\
102        buildlog_filepath: str = '',\
103        generate_warning: bool = True):
104        """Constructor
105
106        @param elf_filename (str) Path to the output binary
107            parsed by objdump to determine section sizes.
108        @param extra_sections (list[str]) List of extra,
109            unexpected sections, which Twister should not
110            report as error and not include in the
111            size calculation.
112        @param buildlog_filepath (str, default: '') Path to the
113            output build.log
114        @param generate_warning (bool, default: True) Twister should
115            (or not) warning about missing the information about
116            footprint.
117        """
118        self.elf_filename = elf_filename
119        self.buildlog_filename = buildlog_filepath
120        self.sections = []
121        self.used_rom = 0
122        self.used_ram = 0
123        self.available_ram = 0
124        self.available_rom = 0
125        self.extra_sections = extra_sections
126        self.is_xip = True
127        self.generate_warning = generate_warning
128
129        self._calculate_sizes()
130
131    def size_report(self):
132        print(self.elf_filename)
133        print("SECTION NAME             VMA        LMA     SIZE  HEX SZ TYPE")
134        for v in self.sections:
135            print(
136                f'{v["name"]:<17} {v["virt_addr"]:#010x} {v["load_addr"]:#010x}'
137                f' {v["size"]:>8} {v["size"]:#07x} {v["type"]:<7}'
138            )
139
140        print(f"Totals: {self.used_rom} bytes (ROM), {self.used_ram} bytes (RAM)")
141        print("")
142
143    def get_used_ram(self):
144        """Get the amount of RAM the application will use up on the device
145
146        @return amount of RAM, in bytes
147        """
148        return self.used_ram
149
150    def get_used_rom(self):
151        """Get the size of the data that this application uses on device's flash
152
153        @return amount of ROM, in bytes
154        """
155        return self.used_rom
156
157    def unrecognized_sections(self):
158        """Get a list of sections inside the binary that weren't recognized
159
160        @return list of unrecognized section names
161        """
162        slist = []
163        for v in self.sections:
164            if not v["recognized"]:
165                slist.append(v["name"])
166        return slist
167
168    def get_available_ram(self) -> int:
169        """Get the total available RAM.
170
171        @return total available RAM, in bytes (int)
172        """
173        return self.available_ram
174
175    def get_available_rom(self) -> int:
176        """Get the total available ROM.
177
178        @return total available ROM, in bytes (int)
179        """
180        return self.available_rom
181
182    def _calculate_sizes(self):
183        """ELF file is analyzed, even if the option to read memory
184        footprint from the build.log file is set.
185        This is to detect potential problems contained in
186        unrecognized sections of the file.
187        """
188        self._analyze_elf_file()
189        if self.buildlog_filename.endswith("build.log"):
190            self._get_footprint_from_buildlog()
191
192    def _check_elf_file(self) -> None:
193        # Make sure this is an ELF binary
194        with open(self.elf_filename, "rb") as f:
195            magic = f.read(4)
196
197        try:
198            if magic != b'\x7fELF':
199                raise TwisterRuntimeError(f"{self.elf_filename} is not an ELF binary")
200        except Exception as e:
201            print(str(e))
202            sys.exit(2)
203
204    def _check_is_xip(self) -> None:
205        # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
206        # GREP can not be used as it returns an error if the symbol is not
207        # found.
208        is_xip_command = "nm " + self.elf_filename + \
209                         " | awk '/CONFIG_XIP/ { print $3 }'"
210        is_xip_output = subprocess.check_output(
211            is_xip_command, shell=True, stderr=subprocess.STDOUT).decode(
212            "utf-8").strip()
213        try:
214            if is_xip_output.endswith("no symbols"):
215                raise TwisterRuntimeError(f"{self.elf_filename} has no symbol information")
216        except Exception as e:
217            print(str(e))
218            sys.exit(2)
219
220        self.is_xip = len(is_xip_output) != 0
221
222    def _get_info_elf_sections(self) -> None:
223        """Calculate RAM and ROM usage and information about issues by section"""
224        objdump_command = "objdump -h " + self.elf_filename
225        objdump_output = subprocess.check_output(
226            objdump_command, shell=True).decode("utf-8").splitlines()
227
228        for line in objdump_output:
229            words = line.split()
230
231            if not words:  # Skip lines that are too short
232                continue
233
234            index = words[0]
235            if not index[0].isdigit():  # Skip lines that do not start
236                continue  # with a digit
237
238            name = words[1]  # Skip lines with section names
239            if name[0] == '.':  # starting with '.'
240                continue
241
242            # TODO this doesn't actually reflect the size in flash or RAM as
243            # it doesn't include linker-imposed padding between sections.
244            # It is close though.
245            size = int(words[2], 16)
246            if size == 0:
247                continue
248
249            load_addr = int(words[4], 16)
250            virt_addr = int(words[3], 16)
251
252            # Add section to memory use totals (for both non-XIP and XIP scenarios)
253            # Unrecognized section names are not included in the calculations.
254            recognized = True
255
256            # If build.log file exists, check errors (unrecognized sections
257            # in ELF file).
258            if self.buildlog_filename:
259                if name in SizeCalculator.alloc_sections or \
260                   name in SizeCalculator.rw_sections or \
261                   name in SizeCalculator.ro_sections:
262                    continue
263                else:
264                    stype = "unknown"
265                    if name not in self.extra_sections:
266                        recognized = False
267            else:
268                if name in SizeCalculator.alloc_sections:
269                    self.used_ram += size
270                    stype = "alloc"
271                elif name in SizeCalculator.rw_sections:
272                    self.used_ram += size
273                    self.used_rom += size
274                    stype = "rw"
275                elif name in SizeCalculator.ro_sections:
276                    self.used_rom += size
277                    if not self.is_xip:
278                        self.used_ram += size
279                    stype = "ro"
280                else:
281                    stype = "unknown"
282                    if name not in self.extra_sections:
283                        recognized = False
284
285            self.sections.append({"name": name, "load_addr": load_addr,
286                                  "size": size, "virt_addr": virt_addr,
287                                  "type": stype, "recognized": recognized})
288
289    def _analyze_elf_file(self) -> None:
290        self._check_elf_file()
291        self._check_is_xip()
292        self._get_info_elf_sections()
293
294    def _get_buildlog_file_content(self) -> list[str]:
295        """Get content of the build.log file.
296
297        @return Content of the build.log file (list[str])
298        """
299        if os.path.exists(path=self.buildlog_filename):
300            with open(file=self.buildlog_filename) as file:
301                file_content = file.readlines()
302        else:
303            if self.generate_warning:
304                logger.error(
305                    msg="Incorrect path to build.log file to analyze footprints."
306                       f" Please check the path {self.buildlog_filename}."
307                )
308            file_content = []
309        return file_content
310
311    def _find_offset_of_last_pattern_occurrence(self, file_content: list[str]) -> int:
312        """Find the offset from which the information about the memory footprint is read.
313
314        @param file_content (list[str]) Content of build.log.
315        @return Offset with information about the memory footprint (int)
316        """
317        result = -1
318        if len(file_content) == 0:
319            logger.warning("Build.log file is empty.")
320        else:
321            # Pattern to first line with information about memory footprint
322            PATTERN_SEARCHED_LINE = "Memory region"
323            # Check the file in reverse order.
324            for idx, line in enumerate(reversed(file_content)):
325                # Condition is fulfilled if the pattern matches with the start of the line.
326                if re.match(pattern=PATTERN_SEARCHED_LINE, string=line):
327                    result = idx + 1
328                    break
329        # If the file does not contain information about memory footprint, the warning is raised.
330        if result == -1:
331            logger.warning(
332                msg="Information about memory footprint for this test configuration is not found."
333                   f" Please check file {self.buildlog_filename}."
334            )
335        return result
336
337    def _get_lines_with_footprint(self, start_offset: int, file_content: list[str]) -> list[str]:
338        """Get lines from the file with a memory footprint.
339
340        @param start_offset (int) Offset with the memory footprint's first line.
341        @param file_content (list[str]) Content of the build.log file.
342        @return Lines with information about memory footprint (list[str])
343        """
344        if len(file_content) == 0:
345            result = []
346        else:
347            if start_offset > len(file_content) or start_offset <= 0:
348                info_line_idx_start = len(file_content) - 1
349            else:
350                info_line_idx_start = len(file_content) - start_offset
351
352            info_line_idx_stop = info_line_idx_start + self.USEFUL_LINES_AMOUNT
353            if info_line_idx_stop > len(file_content):
354                info_line_idx_stop = len(file_content)
355
356            result = file_content[info_line_idx_start:info_line_idx_stop]
357        return result
358
359    def _clear_whitespaces_from_lines(self, text_lines: list[str]) -> list[str]:
360        """Clear text lines from whitespaces.
361
362        @param text_lines (list[str]) Lines with useful information.
363        @return  Cleared text lines with information (list[str])
364        """
365        return [line.strip("\n").rstrip("%") for line in text_lines] if text_lines else []
366
367    def _divide_text_lines_into_columns(self, text_lines: list[str]) -> list[list[str]]:
368        """Divide lines of text into columns.
369
370        @param lines (list[list[str]]) Lines with information about memory footprint.
371        @return Lines divided into columns (list[list[str]])
372        """
373        if text_lines:
374            result = []
375            PATTERN_SPLIT_COLUMNS = "  +"
376            for line in text_lines:
377                line = [
378                    column.rstrip(":") for column in re.split(
379                        pattern=PATTERN_SPLIT_COLUMNS,
380                        string=line
381                    )
382                ]
383                result.append(list(filter(None, line)))
384        else:
385            result = [[]]
386
387        return result
388
389    def _unify_prefixes_on_all_values(self, data_lines: list[list[str]]) -> list[list[str]]:
390        """Convert all values in the table to unified order of magnitude.
391
392        @param data_lines (list[list[str]]) Lines with information about memory footprint.
393        @return Lines with unified values (list[list[str]])
394        """
395        if len(data_lines) != self.USEFUL_LINES_AMOUNT:
396            data_lines = [[]]
397            if self.generate_warning:
398                logger.warning(
399                    msg="Incomplete information about memory footprint."
400                       f" Please check file {self.buildlog_filename}"
401                )
402        else:
403            for idx, line in enumerate(data_lines):
404                # Line with description of the columns
405                if idx == 0:
406                    continue
407                line_to_replace = list(map(self._binary_prefix_converter, line))
408                data_lines[idx] = line_to_replace
409
410        return data_lines
411
412    def _binary_prefix_converter(self, value: str) -> str:
413        """Convert specific value to particular prefix.
414
415        @param value (str) Value to convert.
416        @return Converted value to output prefix (str)
417        """
418        PATTERN_VALUE = r"([0-9]?\s.?B\Z)"
419
420        if not re.search(pattern=PATTERN_VALUE, string=value):
421            converted_value = value.rstrip()
422        else:
423            PREFIX_POWER = {'B': 0, 'KB': 10, 'MB': 20, 'GB': 30}
424            DEFAULT_DATA_PREFIX = 'B'
425
426            data_parts = value.split()
427            numeric_value = int(data_parts[0])
428            unit = data_parts[1]
429            shift = PREFIX_POWER.get(unit, 0) - PREFIX_POWER.get(DEFAULT_DATA_PREFIX, 0)
430            unit_predictor = pow(2, shift)
431            converted_value = str(numeric_value * unit_predictor)
432        return converted_value
433
434    def _create_data_table(self) -> list[list[str]]:
435        """Create table with information about memory footprint.
436
437        @return Table with information about memory usage (list[list[str]])
438        """
439        file_content = self._get_buildlog_file_content()
440        data_line_start_idx = self._find_offset_of_last_pattern_occurrence(
441            file_content=file_content
442        )
443
444        if data_line_start_idx < 0:
445            data_from_content = [[]]
446        else:
447            # Clean lines and separate information to columns
448            information_lines = self._get_lines_with_footprint(
449                start_offset=data_line_start_idx,
450                file_content=file_content
451            )
452            information_lines = self._clear_whitespaces_from_lines(text_lines=information_lines)
453            data_from_content = self._divide_text_lines_into_columns(text_lines=information_lines)
454            data_from_content = self._unify_prefixes_on_all_values(data_lines=data_from_content)
455
456        return data_from_content
457
458    def _get_footprint_from_buildlog(self) -> None:
459        """Get memory footprint from build.log"""
460        data_from_file = self._create_data_table()
461
462        if data_from_file == [[]] or not data_from_file:
463            self.used_ram = 0
464            self.used_rom = 0
465            self.available_ram = 0
466            self.available_rom = 0
467            if self.generate_warning:
468                logger.warning(
469                    msg="Missing information about memory footprint."
470                       f" Check file {self.buildlog_filename}."
471                )
472        else:
473            ROW_RAM_IDX = 2
474            ROW_ROM_IDX = 1
475            COLUMN_USED_SIZE_IDX = 1
476            COLUMN_AVAILABLE_SIZE_IDX = 2
477
478            self.used_ram = int(data_from_file[ROW_RAM_IDX][COLUMN_USED_SIZE_IDX])
479            self.used_rom = int(data_from_file[ROW_ROM_IDX][COLUMN_USED_SIZE_IDX])
480            self.available_ram = int(data_from_file[ROW_RAM_IDX][COLUMN_AVAILABLE_SIZE_IDX])
481            self.available_rom = int(data_from_file[ROW_ROM_IDX][COLUMN_AVAILABLE_SIZE_IDX])
482