1#!/usr/bin/env python3
2
3# SPDX-License-Identifier: Apache-2.0
4"""
5This script help you to compare footprint results with previous commits in git.
6If you don't have a git repository, it will compare your current tree
7against the last release results.
8To run it you need to set up the same environment as twister.
9The scripts take 2 optional args COMMIT and BASE_COMMIT, which tell the scripts
10which commit to use as current commit and as base for comparing, respectively.
11The script can take any SHA commit recognized for git.
12
13COMMIT is the commit to compare against BASE_COMMIT.
14 Default
15 current working directory if we have changes in git tree or we don't have git.
16 HEAD in any other case.
17BASE_COMMIT is the commit used as base to compare results.
18 Default:
19  twister_last_release.csv if we don't have git tree.
20  HEAD is we have changes in the working tree.
21  HEAD~1 if we don't have changes and we have default COMMIT.
22  COMMIT~1 if we have a valid COMMIT.
23
24"""
25
26import argparse
27import os
28import csv
29import subprocess
30import logging
31import tempfile
32import shutil
33
34if "ZEPHYR_BASE" not in os.environ:
35    logging.error("$ZEPHYR_BASE environment variable undefined.\n")
36    exit(1)
37
38logger = None
39GIT_ENABLED = False
40RELEASE_DATA = 'twister_last_release.csv'
41
42def is_git_enabled():
43    global GIT_ENABLED
44    proc = subprocess.Popen('git rev-parse --is-inside-work-tree',
45                            stdout=subprocess.PIPE,
46                            cwd=os.environ.get('ZEPHYR_BASE'), shell=True)
47    if proc.wait() != 0:
48        GIT_ENABLED = False
49
50    GIT_ENABLED = True
51
52def init_logs():
53    global logger
54    log_lev = os.environ.get('LOG_LEVEL', None)
55    level = logging.INFO
56    if log_lev == "DEBUG":
57        level = logging.DEBUG
58    elif log_lev == "ERROR":
59        level = logging.ERROR
60
61    console = logging.StreamHandler()
62    format = logging.Formatter('%(levelname)-8s: %(message)s')
63    console.setFormatter(format)
64    logger = logging.getLogger('')
65    logger.addHandler(console)
66    logger.setLevel(level)
67
68    logging.debug("Log init completed")
69
70def parse_args():
71    parser = argparse.ArgumentParser(
72                description="Compare footprint apps RAM and ROM sizes. Note: "
73                "To run it you need to set up the same environment as twister.",
74                allow_abbrev=False)
75    parser.add_argument('-b', '--base-commit', default=None,
76                        help="Commit ID to use as base for footprint "
77                                    "compare. Default is parent current commit."
78                                    " or twister_last_release.csv if we don't have git.")
79    parser.add_argument('-c', '--commit', default=None,
80                        help="Commit ID to use compare footprint against base. "
81                                    "Default is HEAD or working tree.")
82    return parser.parse_args()
83
84def get_git_commit(commit):
85    commit_id = None
86    proc = subprocess.Popen('git rev-parse %s' % commit, stdout=subprocess.PIPE,
87                            cwd=os.environ.get('ZEPHYR_BASE'), shell=True)
88    if proc.wait() == 0:
89        commit_id = proc.stdout.read().decode("utf-8").strip()
90    return commit_id
91
92def sanity_results_filename(commit=None, cwd=os.environ.get('ZEPHYR_BASE')):
93    if not commit:
94        file_name = "tmp.csv"
95    else:
96        if commit == RELEASE_DATA:
97            file_name = RELEASE_DATA
98        else:
99            file_name = "%s.csv" % commit
100
101    return os.path.join(cwd,'scripts', 'sanity_chk', file_name)
102
103def git_checkout(commit, cwd=os.environ.get('ZEPHYR_BASE')):
104    proc = subprocess.Popen('git diff --quiet', stdout=subprocess.PIPE,
105                            stderr=subprocess.STDOUT, cwd=cwd, shell=True)
106    if proc.wait() != 0:
107        raise Exception("Cannot continue, you have unstaged changes in your working tree")
108
109    proc = subprocess.Popen('git reset %s --hard' % commit,
110                            stdout=subprocess.PIPE,
111                            stderr=subprocess.STDOUT,
112                            cwd=cwd, shell=True)
113    if proc.wait() == 0:
114        return True
115    else:
116        logger.error(proc.stdout.read())
117    return False
118
119def run_sanity_footprint(commit=None, cwd=os.environ.get('ZEPHYR_BASE'),
120                         output_file=None):
121    if not output_file:
122        output_file = sanity_results_filename(commit)
123    cmd = '/bin/bash -c "source ./zephyr-env.sh && twister'
124    cmd += ' +scripts/sanity_chk/sanity_compare.args -o %s"' % output_file
125    logger.debug('Sanity (%s)   %s' %(commit, cmd))
126
127    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
128                            cwd=cwd, shell=True)
129    output,_ = proc.communicate()
130    if proc.wait() == 0:
131        logger.debug(output)
132        return True
133
134    logger.error("Couldn't build footprint apps in commit %s" % commit)
135    logger.error(output)
136    raise Exception("Couldn't build footprint apps in commit %s" % commit)
137
138def run_footprint_build(commit=None):
139    logging.debug("footprint build for %s" % commit)
140    if not commit:
141        run_sanity_footprint()
142    else:
143        cmd = "git clone --no-hardlinks %s" % os.environ.get('ZEPHYR_BASE')
144        tmp_location = os.path.join(tempfile.gettempdir(),
145                                os.path.basename(os.environ.get('ZEPHYR_BASE')))
146        if os.path.exists(tmp_location):
147            shutil.rmtree(tmp_location)
148        logging.debug("cloning into %s" % tmp_location)
149        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
150                                stderr=subprocess.STDOUT,
151                                cwd=tempfile.gettempdir(), shell=True)
152        if proc.wait() == 0:
153            if git_checkout(commit, tmp_location):
154                run_sanity_footprint(commit, tmp_location)
155        else:
156            logger.error(proc.stdout.read())
157        shutil.rmtree(tmp_location, ignore_errors=True)
158    return True
159
160def read_sanity_report(filename):
161    data = []
162    with open(filename) as fp:
163        tmp = csv.DictReader(fp)
164        for row in tmp:
165            data.append(row)
166    return data
167
168def get_footprint_results(commit=None):
169    sanity_file = sanity_results_filename(commit)
170    if (not os.path.exists(sanity_file) or not commit) and commit != RELEASE_DATA:
171        run_footprint_build(commit)
172
173    return read_sanity_report(sanity_file)
174
175def tree_changes():
176    proc = subprocess.Popen('git diff --quiet', stdout=subprocess.PIPE,
177                            cwd=os.environ.get('ZEPHYR_BASE'), shell=True)
178    if proc.wait() != 0:
179        return True
180    return False
181
182def get_default_current_commit():
183    if tree_changes():
184        return None
185    else:
186        return get_git_commit('HEAD')
187
188def get_default_base_commit(current_commit):
189    if not current_commit:
190        if tree_changes():
191            return get_git_commit('HEAD')
192        else:
193            return get_git_commit('HEAD~1')
194    else:
195        return get_git_commit('%s~1'%current_commit)
196
197def build_history(b_commit=None, c_commit=None):
198    if not GIT_ENABLED:
199        logger.info('Working on current tree, not git enabled.')
200        current_commit = None
201        base_commit = RELEASE_DATA
202    else:
203        if not c_commit:
204            current_commit = get_default_current_commit()
205        else:
206            current_commit = get_git_commit(c_commit)
207
208        if not b_commit:
209            base_commit = get_default_base_commit(current_commit)
210        else:
211            base_commit = get_git_commit(b_commit)
212
213    if not base_commit:
214        logger.error("Cannot resolve base commit")
215        return
216
217    logger.info("Base:    %s" % base_commit)
218    logger.info("Current: %s" % (current_commit if current_commit else
219                    'working space'))
220
221    current_results = get_footprint_results(current_commit)
222    base_results = get_footprint_results(base_commit)
223    deltas = compare_results(base_results, current_results)
224    print_deltas(deltas)
225
226def compare_results(base_results, current_results):
227    interesting_metrics = [("ram_size", int),
228                           ("rom_size", int)]
229    results = {}
230    metrics = {}
231
232    for type, data in {'base': base_results, 'current': current_results}.items():
233        metrics[type] = {}
234        for row in data:
235            d = {}
236            for m, mtype in interesting_metrics:
237                if row[m]:
238                    d[m] = mtype(row[m])
239            if not row["test"] in metrics[type]:
240                metrics[type][row["test"]] = {}
241            metrics[type][row["test"]][row["platform"]] = d
242
243    for test, platforms in metrics['current'].items():
244        if not test in metrics['base']:
245            continue
246        tests = {}
247
248        for platform, test_data in platforms.items():
249            if not platform in metrics['base'][test]:
250                continue
251            golden_metric = metrics['base'][test][platform]
252            tmp = {}
253            for metric, _ in interesting_metrics:
254                if metric not in golden_metric or metric not in test_data:
255                    continue
256                if test_data[metric] == "":
257                    continue
258                delta = test_data[metric] - golden_metric[metric]
259                if delta == 0:
260                    continue
261                tmp[metric] = {
262                    'delta': delta,
263                    'current': test_data[metric],
264                }
265
266            if tmp:
267                tests[platform] = tmp
268
269        if tests:
270            results[test] = tests
271
272    return results
273
274def print_deltas(deltas):
275    error_count = 0
276    for test in sorted(deltas):
277        print("\n{:<25}".format(test))
278        for platform, data in deltas[test].items():
279            print(" {:<25}".format(platform))
280            for metric, value in data.items():
281                percentage = (float(value['delta']) / float(value['current'] -
282                                                            value['delta']))
283                print("  {} ({:+.2%}) {:+6}   current size {:>7} bytes".format(
284                        "RAM" if metric == "ram_size" else "ROM", percentage,
285                        value['delta'], value['current']))
286                error_count = error_count + 1
287    if error_count == 0:
288        print("There are no changes in RAM neither in ROM of footprint apps.")
289    return error_count
290
291def main():
292    args = parse_args()
293    build_history(args.base_commit, args.commit)
294
295if __name__ == "__main__":
296    init_logs()
297    is_git_enabled()
298    main()
299