1#!/usr/bin/python3
2
3# Copyright (C) 2017 Netronome Systems, Inc.
4#
5# This software is licensed under the GNU General License Version 2,
6# June 1991 as shown in the file COPYING in the top-level directory of this
7# source tree.
8#
9# THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
10# WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
11# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
12# FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
13# OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
14# THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
15
16from datetime import datetime
17import argparse
18import json
19import os
20import pprint
21import random
22import string
23import struct
24import subprocess
25import time
26
27logfile = None
28log_level = 1
29skip_extack = False
30bpf_test_dir = os.path.dirname(os.path.realpath(__file__))
31pp = pprint.PrettyPrinter()
32devs = [] # devices we created for clean up
33files = [] # files to be removed
34netns = [] # net namespaces to be removed
35
36def log_get_sec(level=0):
37    return "*" * (log_level + level)
38
39def log_level_inc(add=1):
40    global log_level
41    log_level += add
42
43def log_level_dec(sub=1):
44    global log_level
45    log_level -= sub
46
47def log_level_set(level):
48    global log_level
49    log_level = level
50
51def log(header, data, level=None):
52    """
53    Output to an optional log.
54    """
55    if logfile is None:
56        return
57    if level is not None:
58        log_level_set(level)
59
60    if not isinstance(data, str):
61        data = pp.pformat(data)
62
63    if len(header):
64        logfile.write("\n" + log_get_sec() + " ")
65        logfile.write(header)
66    if len(header) and len(data.strip()):
67        logfile.write("\n")
68    logfile.write(data)
69
70def skip(cond, msg):
71    if not cond:
72        return
73    print("SKIP: " + msg)
74    log("SKIP: " + msg, "", level=1)
75    os.sys.exit(0)
76
77def fail(cond, msg):
78    if not cond:
79        return
80    print("FAIL: " + msg)
81    log("FAIL: " + msg, "", level=1)
82    os.sys.exit(1)
83
84def start_test(msg):
85    log(msg, "", level=1)
86    log_level_inc()
87    print(msg)
88
89def cmd(cmd, shell=True, include_stderr=False, background=False, fail=True):
90    """
91    Run a command in subprocess and return tuple of (retval, stdout);
92    optionally return stderr as well as third value.
93    """
94    proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE,
95                            stderr=subprocess.PIPE)
96    if background:
97        msg = "%s START: %s" % (log_get_sec(1),
98                                datetime.now().strftime("%H:%M:%S.%f"))
99        log("BKG " + proc.args, msg)
100        return proc
101
102    return cmd_result(proc, include_stderr=include_stderr, fail=fail)
103
104def cmd_result(proc, include_stderr=False, fail=False):
105    stdout, stderr = proc.communicate()
106    stdout = stdout.decode("utf-8")
107    stderr = stderr.decode("utf-8")
108    proc.stdout.close()
109    proc.stderr.close()
110
111    stderr = "\n" + stderr
112    if stderr[-1] == "\n":
113        stderr = stderr[:-1]
114
115    sec = log_get_sec(1)
116    log("CMD " + proc.args,
117        "RETCODE: %d\n%s STDOUT:\n%s%s STDERR:%s\n%s END: %s" %
118        (proc.returncode, sec, stdout, sec, stderr,
119         sec, datetime.now().strftime("%H:%M:%S.%f")))
120
121    if proc.returncode != 0 and fail:
122        if len(stderr) > 0 and stderr[-1] == "\n":
123            stderr = stderr[:-1]
124        raise Exception("Command failed: %s\n%s" % (proc.args, stderr))
125
126    if include_stderr:
127        return proc.returncode, stdout, stderr
128    else:
129        return proc.returncode, stdout
130
131def rm(f):
132    cmd("rm -f %s" % (f))
133    if f in files:
134        files.remove(f)
135
136def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False):
137    params = ""
138    if JSON:
139        params += "%s " % (flags["json"])
140
141    if ns != "":
142        ns = "ip netns exec %s " % (ns)
143
144    if include_stderr:
145        ret, stdout, stderr = cmd(ns + name + " " + params + args,
146                                  fail=fail, include_stderr=True)
147    else:
148        ret, stdout = cmd(ns + name + " " + params + args,
149                          fail=fail, include_stderr=False)
150
151    if JSON and len(stdout.strip()) != 0:
152        out = json.loads(stdout)
153    else:
154        out = stdout
155
156    if include_stderr:
157        return ret, out, stderr
158    else:
159        return ret, out
160
161def bpftool(args, JSON=True, ns="", fail=True, include_stderr=False):
162    return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns,
163                fail=fail, include_stderr=include_stderr)
164
165def bpftool_prog_list(expected=None, ns=""):
166    _, progs = bpftool("prog show", JSON=True, ns=ns, fail=True)
167    # Remove the base progs
168    for p in base_progs:
169        if p in progs:
170            progs.remove(p)
171    if expected is not None:
172        if len(progs) != expected:
173            fail(True, "%d BPF programs loaded, expected %d" %
174                 (len(progs), expected))
175    return progs
176
177def bpftool_map_list(expected=None, ns=""):
178    _, maps = bpftool("map show", JSON=True, ns=ns, fail=True)
179    # Remove the base maps
180    for m in base_maps:
181        if m in maps:
182            maps.remove(m)
183    if expected is not None:
184        if len(maps) != expected:
185            fail(True, "%d BPF maps loaded, expected %d" %
186                 (len(maps), expected))
187    return maps
188
189def bpftool_prog_list_wait(expected=0, n_retry=20):
190    for i in range(n_retry):
191        nprogs = len(bpftool_prog_list())
192        if nprogs == expected:
193            return
194        time.sleep(0.05)
195    raise Exception("Time out waiting for program counts to stabilize want %d, have %d" % (expected, nprogs))
196
197def bpftool_map_list_wait(expected=0, n_retry=20):
198    for i in range(n_retry):
199        nmaps = len(bpftool_map_list())
200        if nmaps == expected:
201            return
202        time.sleep(0.05)
203    raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps))
204
205def bpftool_prog_load(sample, file_name, maps=[], prog_type="xdp", dev=None,
206                      fail=True, include_stderr=False):
207    args = "prog load %s %s" % (os.path.join(bpf_test_dir, sample), file_name)
208    if prog_type is not None:
209        args += " type " + prog_type
210    if dev is not None:
211        args += " dev " + dev
212    if len(maps):
213        args += " map " + " map ".join(maps)
214
215    res = bpftool(args, fail=fail, include_stderr=include_stderr)
216    if res[0] == 0:
217        files.append(file_name)
218    return res
219
220def ip(args, force=False, JSON=True, ns="", fail=True, include_stderr=False):
221    if force:
222        args = "-force " + args
223    return tool("ip", args, {"json":"-j"}, JSON=JSON, ns=ns,
224                fail=fail, include_stderr=include_stderr)
225
226def tc(args, JSON=True, ns="", fail=True, include_stderr=False):
227    return tool("tc", args, {"json":"-p"}, JSON=JSON, ns=ns,
228                fail=fail, include_stderr=include_stderr)
229
230def ethtool(dev, opt, args, fail=True):
231    return cmd("ethtool %s %s %s" % (opt, dev["ifname"], args), fail=fail)
232
233def bpf_obj(name, sec=".text", path=bpf_test_dir,):
234    return "obj %s sec %s" % (os.path.join(path, name), sec)
235
236def bpf_pinned(name):
237    return "pinned %s" % (name)
238
239def bpf_bytecode(bytecode):
240    return "bytecode \"%s\"" % (bytecode)
241
242def mknetns(n_retry=10):
243    for i in range(n_retry):
244        name = ''.join([random.choice(string.ascii_letters) for i in range(8)])
245        ret, _ = ip("netns add %s" % (name), fail=False)
246        if ret == 0:
247            netns.append(name)
248            return name
249    return None
250
251def int2str(fmt, val):
252    ret = []
253    for b in struct.pack(fmt, val):
254        ret.append(int(b))
255    return " ".join(map(lambda x: str(x), ret))
256
257def str2int(strtab):
258    inttab = []
259    for i in strtab:
260        inttab.append(int(i, 16))
261    ba = bytearray(inttab)
262    if len(strtab) == 4:
263        fmt = "I"
264    elif len(strtab) == 8:
265        fmt = "Q"
266    else:
267        raise Exception("String array of len %d can't be unpacked to an int" %
268                        (len(strtab)))
269    return struct.unpack(fmt, ba)[0]
270
271class DebugfsDir:
272    """
273    Class for accessing DebugFS directories as a dictionary.
274    """
275
276    def __init__(self, path):
277        self.path = path
278        self._dict = self._debugfs_dir_read(path)
279
280    def __len__(self):
281        return len(self._dict.keys())
282
283    def __getitem__(self, key):
284        if type(key) is int:
285            key = list(self._dict.keys())[key]
286        return self._dict[key]
287
288    def __setitem__(self, key, value):
289        log("DebugFS set %s = %s" % (key, value), "")
290        log_level_inc()
291
292        cmd("echo '%s' > %s/%s" % (value, self.path, key))
293        log_level_dec()
294
295        _, out = cmd('cat %s/%s' % (self.path, key))
296        self._dict[key] = out.strip()
297
298    def _debugfs_dir_read(self, path):
299        dfs = {}
300
301        log("DebugFS state for %s" % (path), "")
302        log_level_inc(add=2)
303
304        _, out = cmd('ls ' + path)
305        for f in out.split():
306            p = os.path.join(path, f)
307            if os.path.isfile(p):
308                _, out = cmd('cat %s/%s' % (path, f))
309                dfs[f] = out.strip()
310            elif os.path.isdir(p):
311                dfs[f] = DebugfsDir(p)
312            else:
313                raise Exception("%s is neither file nor directory" % (p))
314
315        log_level_dec()
316        log("DebugFS state", dfs)
317        log_level_dec()
318
319        return dfs
320
321class NetdevSim:
322    """
323    Class for netdevsim netdevice and its attributes.
324    """
325
326    def __init__(self, link=None):
327        self.link = link
328
329        self.dev = self._netdevsim_create()
330        devs.append(self)
331
332        self.ns = ""
333
334        self.dfs_dir = '/sys/kernel/debug/netdevsim/%s' % (self.dev['ifname'])
335        self.sdev_dir = self.dfs_dir + '/sdev/'
336        self.dfs_refresh()
337
338    def __getitem__(self, key):
339        return self.dev[key]
340
341    def _netdevsim_create(self):
342        link = "" if self.link is None else "link " + self.link.dev['ifname']
343        _, old  = ip("link show")
344        ip("link add sim%d {link} type netdevsim".format(link=link))
345        _, new  = ip("link show")
346
347        for dev in new:
348            f = filter(lambda x: x["ifname"] == dev["ifname"], old)
349            if len(list(f)) == 0:
350                return dev
351
352        raise Exception("failed to create netdevsim device")
353
354    def remove(self):
355        devs.remove(self)
356        ip("link del dev %s" % (self.dev["ifname"]), ns=self.ns)
357
358    def dfs_refresh(self):
359        self.dfs = DebugfsDir(self.dfs_dir)
360        return self.dfs
361
362    def dfs_read(self, f):
363        path = os.path.join(self.dfs_dir, f)
364        _, data = cmd('cat %s' % (path))
365        return data.strip()
366
367    def dfs_num_bound_progs(self):
368        path = os.path.join(self.sdev_dir, "bpf_bound_progs")
369        _, progs = cmd('ls %s' % (path))
370        return len(progs.split())
371
372    def dfs_get_bound_progs(self, expected):
373        progs = DebugfsDir(os.path.join(self.sdev_dir, "bpf_bound_progs"))
374        if expected is not None:
375            if len(progs) != expected:
376                fail(True, "%d BPF programs bound, expected %d" %
377                     (len(progs), expected))
378        return progs
379
380    def wait_for_flush(self, bound=0, total=0, n_retry=20):
381        for i in range(n_retry):
382            nbound = self.dfs_num_bound_progs()
383            nprogs = len(bpftool_prog_list())
384            if nbound == bound and nprogs == total:
385                return
386            time.sleep(0.05)
387        raise Exception("Time out waiting for program counts to stabilize want %d/%d, have %d bound, %d loaded" % (bound, total, nbound, nprogs))
388
389    def set_ns(self, ns):
390        name = "1" if ns == "" else ns
391        ip("link set dev %s netns %s" % (self.dev["ifname"], name), ns=self.ns)
392        self.ns = ns
393
394    def set_mtu(self, mtu, fail=True):
395        return ip("link set dev %s mtu %d" % (self.dev["ifname"], mtu),
396                  fail=fail)
397
398    def set_xdp(self, bpf, mode, force=False, JSON=True, verbose=False,
399                fail=True, include_stderr=False):
400        if verbose:
401            bpf += " verbose"
402        return ip("link set dev %s xdp%s %s" % (self.dev["ifname"], mode, bpf),
403                  force=force, JSON=JSON,
404                  fail=fail, include_stderr=include_stderr)
405
406    def unset_xdp(self, mode, force=False, JSON=True,
407                  fail=True, include_stderr=False):
408        return ip("link set dev %s xdp%s off" % (self.dev["ifname"], mode),
409                  force=force, JSON=JSON,
410                  fail=fail, include_stderr=include_stderr)
411
412    def ip_link_show(self, xdp):
413        _, link = ip("link show dev %s" % (self['ifname']))
414        if len(link) > 1:
415            raise Exception("Multiple objects on ip link show")
416        if len(link) < 1:
417            return {}
418        fail(xdp != "xdp" in link,
419             "XDP program not reporting in iplink (reported %s, expected %s)" %
420             ("xdp" in link, xdp))
421        return link[0]
422
423    def tc_add_ingress(self):
424        tc("qdisc add dev %s ingress" % (self['ifname']))
425
426    def tc_del_ingress(self):
427        tc("qdisc del dev %s ingress" % (self['ifname']))
428
429    def tc_flush_filters(self, bound=0, total=0):
430        self.tc_del_ingress()
431        self.tc_add_ingress()
432        self.wait_for_flush(bound=bound, total=total)
433
434    def tc_show_ingress(self, expected=None):
435        # No JSON support, oh well...
436        flags = ["skip_sw", "skip_hw", "in_hw"]
437        named = ["protocol", "pref", "chain", "handle", "id", "tag"]
438
439        args = "-s filter show dev %s ingress" % (self['ifname'])
440        _, out = tc(args, JSON=False)
441
442        filters = []
443        lines = out.split('\n')
444        for line in lines:
445            words = line.split()
446            if "handle" not in words:
447                continue
448            fltr = {}
449            for flag in flags:
450                fltr[flag] = flag in words
451            for name in named:
452                try:
453                    idx = words.index(name)
454                    fltr[name] = words[idx + 1]
455                except ValueError:
456                    pass
457            filters.append(fltr)
458
459        if expected is not None:
460            fail(len(filters) != expected,
461                 "%d ingress filters loaded, expected %d" %
462                 (len(filters), expected))
463        return filters
464
465    def cls_filter_op(self, op, qdisc="ingress", prio=None, handle=None,
466                      chain=None, cls="", params="",
467                      fail=True, include_stderr=False):
468        spec = ""
469        if prio is not None:
470            spec += " prio %d" % (prio)
471        if handle:
472            spec += " handle %s" % (handle)
473        if chain is not None:
474            spec += " chain %d" % (chain)
475
476        return tc("filter {op} dev {dev} {qdisc} {spec} {cls} {params}"\
477                  .format(op=op, dev=self['ifname'], qdisc=qdisc, spec=spec,
478                          cls=cls, params=params),
479                  fail=fail, include_stderr=include_stderr)
480
481    def cls_bpf_add_filter(self, bpf, op="add", prio=None, handle=None,
482                           chain=None, da=False, verbose=False,
483                           skip_sw=False, skip_hw=False,
484                           fail=True, include_stderr=False):
485        cls = "bpf " + bpf
486
487        params = ""
488        if da:
489            params += " da"
490        if verbose:
491            params += " verbose"
492        if skip_sw:
493            params += " skip_sw"
494        if skip_hw:
495            params += " skip_hw"
496
497        return self.cls_filter_op(op=op, prio=prio, handle=handle, cls=cls,
498                                  chain=chain, params=params,
499                                  fail=fail, include_stderr=include_stderr)
500
501    def set_ethtool_tc_offloads(self, enable, fail=True):
502        args = "hw-tc-offload %s" % ("on" if enable else "off")
503        return ethtool(self, "-K", args, fail=fail)
504
505################################################################################
506def clean_up():
507    global files, netns, devs
508
509    for dev in devs:
510        dev.remove()
511    for f in files:
512        cmd("rm -f %s" % (f))
513    for ns in netns:
514        cmd("ip netns delete %s" % (ns))
515    files = []
516    netns = []
517
518def pin_prog(file_name, idx=0):
519    progs = bpftool_prog_list(expected=(idx + 1))
520    prog = progs[idx]
521    bpftool("prog pin id %d %s" % (prog["id"], file_name))
522    files.append(file_name)
523
524    return file_name, bpf_pinned(file_name)
525
526def pin_map(file_name, idx=0, expected=1):
527    maps = bpftool_map_list(expected=expected)
528    m = maps[idx]
529    bpftool("map pin id %d %s" % (m["id"], file_name))
530    files.append(file_name)
531
532    return file_name, bpf_pinned(file_name)
533
534def check_dev_info_removed(prog_file=None, map_file=None):
535    bpftool_prog_list(expected=0)
536    ret, err = bpftool("prog show pin %s" % (prog_file), fail=False)
537    fail(ret == 0, "Showing prog with removed device did not fail")
538    fail(err["error"].find("No such device") == -1,
539         "Showing prog with removed device expected ENODEV, error is %s" %
540         (err["error"]))
541
542    bpftool_map_list(expected=0)
543    ret, err = bpftool("map show pin %s" % (map_file), fail=False)
544    fail(ret == 0, "Showing map with removed device did not fail")
545    fail(err["error"].find("No such device") == -1,
546         "Showing map with removed device expected ENODEV, error is %s" %
547         (err["error"]))
548
549def check_dev_info(other_ns, ns, prog_file=None, map_file=None, removed=False):
550    progs = bpftool_prog_list(expected=1, ns=ns)
551    prog = progs[0]
552
553    fail("dev" not in prog.keys(), "Device parameters not reported")
554    dev = prog["dev"]
555    fail("ifindex" not in dev.keys(), "Device parameters not reported")
556    fail("ns_dev" not in dev.keys(), "Device parameters not reported")
557    fail("ns_inode" not in dev.keys(), "Device parameters not reported")
558
559    if not other_ns:
560        fail("ifname" not in dev.keys(), "Ifname not reported")
561        fail(dev["ifname"] != sim["ifname"],
562             "Ifname incorrect %s vs %s" % (dev["ifname"], sim["ifname"]))
563    else:
564        fail("ifname" in dev.keys(), "Ifname is reported for other ns")
565
566    maps = bpftool_map_list(expected=2, ns=ns)
567    for m in maps:
568        fail("dev" not in m.keys(), "Device parameters not reported")
569        fail(dev != m["dev"], "Map's device different than program's")
570
571def check_extack(output, reference, args):
572    if skip_extack:
573        return
574    lines = output.split("\n")
575    comp = len(lines) >= 2 and lines[1] == 'Error: ' + reference
576    fail(not comp, "Missing or incorrect netlink extack message")
577
578def check_extack_nsim(output, reference, args):
579    check_extack(output, "netdevsim: " + reference, args)
580
581def check_no_extack(res, needle):
582    fail((res[1] + res[2]).count(needle) or (res[1] + res[2]).count("Warning:"),
583         "Found '%s' in command output, leaky extack?" % (needle))
584
585def check_verifier_log(output, reference):
586    lines = output.split("\n")
587    for l in reversed(lines):
588        if l == reference:
589            return
590    fail(True, "Missing or incorrect message from netdevsim in verifier log")
591
592def test_spurios_extack(sim, obj, skip_hw, needle):
593    res = sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=skip_hw,
594                                 include_stderr=True)
595    check_no_extack(res, needle)
596    res = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
597                                 skip_hw=skip_hw, include_stderr=True)
598    check_no_extack(res, needle)
599    res = sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf",
600                            include_stderr=True)
601    check_no_extack(res, needle)
602
603
604# Parse command line
605parser = argparse.ArgumentParser()
606parser.add_argument("--log", help="output verbose log to given file")
607args = parser.parse_args()
608if args.log:
609    logfile = open(args.log, 'w+')
610    logfile.write("# -*-Org-*-")
611
612log("Prepare...", "", level=1)
613log_level_inc()
614
615# Check permissions
616skip(os.getuid() != 0, "test must be run as root")
617
618# Check tools
619ret, progs = bpftool("prog", fail=False)
620skip(ret != 0, "bpftool not installed")
621base_progs = progs
622_, base_maps = bpftool("map")
623
624# Check netdevsim
625ret, out = cmd("modprobe netdevsim", fail=False)
626skip(ret != 0, "netdevsim module could not be loaded")
627
628# Check debugfs
629_, out = cmd("mount")
630if out.find("/sys/kernel/debug type debugfs") == -1:
631    cmd("mount -t debugfs none /sys/kernel/debug")
632
633# Check samples are compiled
634samples = ["sample_ret0.o", "sample_map_ret0.o"]
635for s in samples:
636    ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)
637    skip(ret != 0, "sample %s/%s not found, please compile it" %
638         (bpf_test_dir, s))
639
640# Check if iproute2 is built with libmnl (needed by extack support)
641_, _, err = cmd("tc qdisc delete dev lo handle 0",
642                fail=False, include_stderr=True)
643if err.find("Error: Failed to find qdisc with specified handle.") == -1:
644    print("Warning: no extack message in iproute2 output, libmnl missing?")
645    log("Warning: no extack message in iproute2 output, libmnl missing?", "")
646    skip_extack = True
647
648# Check if net namespaces seem to work
649ns = mknetns()
650skip(ns is None, "Could not create a net namespace")
651cmd("ip netns delete %s" % (ns))
652netns = []
653
654try:
655    obj = bpf_obj("sample_ret0.o")
656    bytecode = bpf_bytecode("1,6 0 0 4294967295,")
657
658    start_test("Test destruction of generic XDP...")
659    sim = NetdevSim()
660    sim.set_xdp(obj, "generic")
661    sim.remove()
662    bpftool_prog_list_wait(expected=0)
663
664    sim = NetdevSim()
665    sim.tc_add_ingress()
666
667    start_test("Test TC non-offloaded...")
668    ret, _ = sim.cls_bpf_add_filter(obj, skip_hw=True, fail=False)
669    fail(ret != 0, "Software TC filter did not load")
670
671    start_test("Test TC non-offloaded isn't getting bound...")
672    ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
673    fail(ret != 0, "Software TC filter did not load")
674    sim.dfs_get_bound_progs(expected=0)
675
676    sim.tc_flush_filters()
677
678    start_test("Test TC offloads are off by default...")
679    ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
680                                         fail=False, include_stderr=True)
681    fail(ret == 0, "TC filter loaded without enabling TC offloads")
682    check_extack(err, "TC offload is disabled on net device.", args)
683    sim.wait_for_flush()
684
685    sim.set_ethtool_tc_offloads(True)
686    sim.dfs["bpf_tc_non_bound_accept"] = "Y"
687
688    start_test("Test TC offload by default...")
689    ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
690    fail(ret != 0, "Software TC filter did not load")
691    sim.dfs_get_bound_progs(expected=0)
692    ingress = sim.tc_show_ingress(expected=1)
693    fltr = ingress[0]
694    fail(not fltr["in_hw"], "Filter not offloaded by default")
695
696    sim.tc_flush_filters()
697
698    start_test("Test TC cBPF bytcode tries offload by default...")
699    ret, _ = sim.cls_bpf_add_filter(bytecode, fail=False)
700    fail(ret != 0, "Software TC filter did not load")
701    sim.dfs_get_bound_progs(expected=0)
702    ingress = sim.tc_show_ingress(expected=1)
703    fltr = ingress[0]
704    fail(not fltr["in_hw"], "Bytecode not offloaded by default")
705
706    sim.tc_flush_filters()
707    sim.dfs["bpf_tc_non_bound_accept"] = "N"
708
709    start_test("Test TC cBPF unbound bytecode doesn't offload...")
710    ret, _, err = sim.cls_bpf_add_filter(bytecode, skip_sw=True,
711                                         fail=False, include_stderr=True)
712    fail(ret == 0, "TC bytecode loaded for offload")
713    check_extack_nsim(err, "netdevsim configured to reject unbound programs.",
714                      args)
715    sim.wait_for_flush()
716
717    start_test("Test non-0 chain offload...")
718    ret, _, err = sim.cls_bpf_add_filter(obj, chain=1, prio=1, handle=1,
719                                         skip_sw=True,
720                                         fail=False, include_stderr=True)
721    fail(ret == 0, "Offloaded a filter to chain other than 0")
722    check_extack(err, "Driver supports only offload of chain 0.", args)
723    sim.tc_flush_filters()
724
725    start_test("Test TC replace...")
726    sim.cls_bpf_add_filter(obj, prio=1, handle=1)
727    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1)
728    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
729
730    sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_sw=True)
731    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_sw=True)
732    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
733
734    sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=True)
735    sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_hw=True)
736    sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
737
738    start_test("Test TC replace bad flags...")
739    for i in range(3):
740        for j in range(3):
741            ret, _ = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
742                                            skip_sw=(j == 1), skip_hw=(j == 2),
743                                            fail=False)
744            fail(bool(ret) != bool(j),
745                 "Software TC incorrect load in replace test, iteration %d" %
746                 (j))
747        sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
748
749    start_test("Test spurious extack from the driver...")
750    test_spurios_extack(sim, obj, False, "netdevsim")
751    test_spurios_extack(sim, obj, True, "netdevsim")
752
753    sim.set_ethtool_tc_offloads(False)
754
755    test_spurios_extack(sim, obj, False, "TC offload is disabled")
756    test_spurios_extack(sim, obj, True, "TC offload is disabled")
757
758    sim.set_ethtool_tc_offloads(True)
759
760    sim.tc_flush_filters()
761
762    start_test("Test TC offloads work...")
763    ret, _, err = sim.cls_bpf_add_filter(obj, verbose=True, skip_sw=True,
764                                         fail=False, include_stderr=True)
765    fail(ret != 0, "TC filter did not load with TC offloads enabled")
766    check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
767
768    start_test("Test TC offload basics...")
769    dfs = sim.dfs_get_bound_progs(expected=1)
770    progs = bpftool_prog_list(expected=1)
771    ingress = sim.tc_show_ingress(expected=1)
772
773    dprog = dfs[0]
774    prog = progs[0]
775    fltr = ingress[0]
776    fail(fltr["skip_hw"], "TC does reports 'skip_hw' on offloaded filter")
777    fail(not fltr["in_hw"], "TC does not report 'in_hw' for offloaded filter")
778    fail(not fltr["skip_sw"], "TC does not report 'skip_sw' back")
779
780    start_test("Test TC offload is device-bound...")
781    fail(str(prog["id"]) != fltr["id"], "Program IDs don't match")
782    fail(prog["tag"] != fltr["tag"], "Program tags don't match")
783    fail(fltr["id"] != dprog["id"], "Program IDs don't match")
784    fail(dprog["state"] != "xlated", "Offloaded program state not translated")
785    fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
786
787    start_test("Test disabling TC offloads is rejected while filters installed...")
788    ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
789    fail(ret == 0, "Driver should refuse to disable TC offloads with filters installed...")
790
791    start_test("Test qdisc removal frees things...")
792    sim.tc_flush_filters()
793    sim.tc_show_ingress(expected=0)
794
795    start_test("Test disabling TC offloads is OK without filters...")
796    ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
797    fail(ret != 0,
798         "Driver refused to disable TC offloads without filters installed...")
799
800    sim.set_ethtool_tc_offloads(True)
801
802    start_test("Test destroying device gets rid of TC filters...")
803    sim.cls_bpf_add_filter(obj, skip_sw=True)
804    sim.remove()
805    bpftool_prog_list_wait(expected=0)
806
807    sim = NetdevSim()
808    sim.set_ethtool_tc_offloads(True)
809
810    start_test("Test destroying device gets rid of XDP...")
811    sim.set_xdp(obj, "offload")
812    sim.remove()
813    bpftool_prog_list_wait(expected=0)
814
815    sim = NetdevSim()
816    sim.set_ethtool_tc_offloads(True)
817
818    start_test("Test XDP prog reporting...")
819    sim.set_xdp(obj, "drv")
820    ipl = sim.ip_link_show(xdp=True)
821    progs = bpftool_prog_list(expected=1)
822    fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
823         "Loaded program has wrong ID")
824
825    start_test("Test XDP prog replace without force...")
826    ret, _ = sim.set_xdp(obj, "drv", fail=False)
827    fail(ret == 0, "Replaced XDP program without -force")
828    sim.wait_for_flush(total=1)
829
830    start_test("Test XDP prog replace with force...")
831    ret, _ = sim.set_xdp(obj, "drv", force=True, fail=False)
832    fail(ret != 0, "Could not replace XDP program with -force")
833    bpftool_prog_list_wait(expected=1)
834    ipl = sim.ip_link_show(xdp=True)
835    progs = bpftool_prog_list(expected=1)
836    fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
837         "Loaded program has wrong ID")
838    fail("dev" in progs[0].keys(),
839         "Device parameters reported for non-offloaded program")
840
841    start_test("Test XDP prog replace with bad flags...")
842    ret, _, err = sim.set_xdp(obj, "generic", force=True,
843                              fail=False, include_stderr=True)
844    fail(ret == 0, "Replaced XDP program with a program in different mode")
845    fail(err.count("File exists") != 1, "Replaced driver XDP with generic")
846    ret, _, err = sim.set_xdp(obj, "", force=True,
847                              fail=False, include_stderr=True)
848    fail(ret == 0, "Replaced XDP program with a program in different mode")
849    check_extack(err, "program loaded with different flags.", args)
850
851    start_test("Test XDP prog remove with bad flags...")
852    ret, _, err = sim.unset_xdp("", force=True,
853                                fail=False, include_stderr=True)
854    fail(ret == 0, "Removed program with a bad mode")
855    check_extack(err, "program loaded with different flags.", args)
856
857    start_test("Test MTU restrictions...")
858    ret, _ = sim.set_mtu(9000, fail=False)
859    fail(ret == 0,
860         "Driver should refuse to increase MTU to 9000 with XDP loaded...")
861    sim.unset_xdp("drv")
862    bpftool_prog_list_wait(expected=0)
863    sim.set_mtu(9000)
864    ret, _, err = sim.set_xdp(obj, "drv", fail=False, include_stderr=True)
865    fail(ret == 0, "Driver should refuse to load program with MTU of 9000...")
866    check_extack_nsim(err, "MTU too large w/ XDP enabled.", args)
867    sim.set_mtu(1500)
868
869    sim.wait_for_flush()
870    start_test("Test non-offload XDP attaching to HW...")
871    bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload")
872    nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
873    ret, _, err = sim.set_xdp(nooffload, "offload",
874                              fail=False, include_stderr=True)
875    fail(ret == 0, "attached non-offloaded XDP program to HW")
876    check_extack_nsim(err, "xdpoffload of non-bound program.", args)
877    rm("/sys/fs/bpf/nooffload")
878
879    start_test("Test offload XDP attaching to drv...")
880    bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",
881                      dev=sim['ifname'])
882    offload = bpf_pinned("/sys/fs/bpf/offload")
883    ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
884    fail(ret == 0, "attached offloaded XDP program to drv")
885    check_extack(err, "using device-bound program without HW_MODE flag is not supported.", args)
886    rm("/sys/fs/bpf/offload")
887    sim.wait_for_flush()
888
889    start_test("Test XDP offload...")
890    _, _, err = sim.set_xdp(obj, "offload", verbose=True, include_stderr=True)
891    ipl = sim.ip_link_show(xdp=True)
892    link_xdp = ipl["xdp"]["prog"]
893    progs = bpftool_prog_list(expected=1)
894    prog = progs[0]
895    fail(link_xdp["id"] != prog["id"], "Loaded program has wrong ID")
896    check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
897
898    start_test("Test XDP offload is device bound...")
899    dfs = sim.dfs_get_bound_progs(expected=1)
900    dprog = dfs[0]
901
902    fail(prog["id"] != link_xdp["id"], "Program IDs don't match")
903    fail(prog["tag"] != link_xdp["tag"], "Program tags don't match")
904    fail(str(link_xdp["id"]) != dprog["id"], "Program IDs don't match")
905    fail(dprog["state"] != "xlated", "Offloaded program state not translated")
906    fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
907
908    start_test("Test removing XDP program many times...")
909    sim.unset_xdp("offload")
910    sim.unset_xdp("offload")
911    sim.unset_xdp("drv")
912    sim.unset_xdp("drv")
913    sim.unset_xdp("")
914    sim.unset_xdp("")
915    bpftool_prog_list_wait(expected=0)
916
917    start_test("Test attempt to use a program for a wrong device...")
918    sim2 = NetdevSim()
919    sim2.set_xdp(obj, "offload")
920    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
921
922    ret, _, err = sim.set_xdp(pinned, "offload",
923                              fail=False, include_stderr=True)
924    fail(ret == 0, "Pinned program loaded for a different device accepted")
925    check_extack_nsim(err, "program bound to different dev.", args)
926    sim2.remove()
927    ret, _, err = sim.set_xdp(pinned, "offload",
928                              fail=False, include_stderr=True)
929    fail(ret == 0, "Pinned program loaded for a removed device accepted")
930    check_extack_nsim(err, "xdpoffload of non-bound program.", args)
931    rm(pin_file)
932    bpftool_prog_list_wait(expected=0)
933
934    start_test("Test multi-attachment XDP - attach...")
935    sim.set_xdp(obj, "offload")
936    xdp = sim.ip_link_show(xdp=True)["xdp"]
937    offloaded = sim.dfs_read("bpf_offloaded_id")
938    fail("prog" not in xdp, "Base program not reported in single program mode")
939    fail(len(ipl["xdp"]["attached"]) != 1,
940         "Wrong attached program count with one program")
941
942    sim.set_xdp(obj, "")
943    two_xdps = sim.ip_link_show(xdp=True)["xdp"]
944    offloaded2 = sim.dfs_read("bpf_offloaded_id")
945
946    fail(two_xdps["mode"] != 4, "Bad mode reported with multiple programs")
947    fail("prog" in two_xdps, "Base program reported in multi program mode")
948    fail(xdp["attached"][0] not in two_xdps["attached"],
949         "Offload program not reported after driver activated")
950    fail(len(two_xdps["attached"]) != 2,
951         "Wrong attached program count with two programs")
952    fail(two_xdps["attached"][0]["prog"]["id"] ==
953         two_xdps["attached"][1]["prog"]["id"],
954         "offloaded and drv programs have the same id")
955    fail(offloaded != offloaded2,
956         "offload ID changed after loading driver program")
957
958    start_test("Test multi-attachment XDP - replace...")
959    ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
960    fail(err.count("busy") != 1, "Replaced one of programs without -force")
961
962    start_test("Test multi-attachment XDP - detach...")
963    ret, _, err = sim.unset_xdp("drv", force=True,
964                                fail=False, include_stderr=True)
965    fail(ret == 0, "Removed program with a bad mode")
966    check_extack(err, "program loaded with different flags.", args)
967
968    sim.unset_xdp("offload")
969    xdp = sim.ip_link_show(xdp=True)["xdp"]
970    offloaded = sim.dfs_read("bpf_offloaded_id")
971
972    fail(xdp["mode"] != 1, "Bad mode reported after multiple programs")
973    fail("prog" not in xdp,
974         "Base program not reported after multi program mode")
975    fail(xdp["attached"][0] not in two_xdps["attached"],
976         "Offload program not reported after driver activated")
977    fail(len(ipl["xdp"]["attached"]) != 1,
978         "Wrong attached program count with remaining programs")
979    fail(offloaded != "0", "offload ID reported with only driver program left")
980
981    start_test("Test multi-attachment XDP - device remove...")
982    sim.set_xdp(obj, "offload")
983    sim.remove()
984
985    sim = NetdevSim()
986    sim.set_ethtool_tc_offloads(True)
987
988    start_test("Test mixing of TC and XDP...")
989    sim.tc_add_ingress()
990    sim.set_xdp(obj, "offload")
991    ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
992                                         fail=False, include_stderr=True)
993    fail(ret == 0, "Loading TC when XDP active should fail")
994    check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
995    sim.unset_xdp("offload")
996    sim.wait_for_flush()
997
998    sim.cls_bpf_add_filter(obj, skip_sw=True)
999    ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
1000    fail(ret == 0, "Loading XDP when TC active should fail")
1001    check_extack_nsim(err, "TC program is already loaded.", args)
1002
1003    start_test("Test binding TC from pinned...")
1004    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
1005    sim.tc_flush_filters(bound=1, total=1)
1006    sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True)
1007    sim.tc_flush_filters(bound=1, total=1)
1008
1009    start_test("Test binding XDP from pinned...")
1010    sim.set_xdp(obj, "offload")
1011    pin_file, pinned = pin_prog("/sys/fs/bpf/tmp2", idx=1)
1012
1013    sim.set_xdp(pinned, "offload", force=True)
1014    sim.unset_xdp("offload")
1015    sim.set_xdp(pinned, "offload", force=True)
1016    sim.unset_xdp("offload")
1017
1018    start_test("Test offload of wrong type fails...")
1019    ret, _ = sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True, fail=False)
1020    fail(ret == 0, "Managed to attach XDP program to TC")
1021
1022    start_test("Test asking for TC offload of two filters...")
1023    sim.cls_bpf_add_filter(obj, da=True, skip_sw=True)
1024    ret, _, err = sim.cls_bpf_add_filter(obj, da=True, skip_sw=True,
1025                                         fail=False, include_stderr=True)
1026    fail(ret == 0, "Managed to offload two TC filters at the same time")
1027    check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
1028
1029    sim.tc_flush_filters(bound=2, total=2)
1030
1031    start_test("Test if netdev removal waits for translation...")
1032    delay_msec = 500
1033    sim.dfs["bpf_bind_verifier_delay"] = delay_msec
1034    start = time.time()
1035    cmd_line = "tc filter add dev %s ingress bpf %s da skip_sw" % \
1036               (sim['ifname'], obj)
1037    tc_proc = cmd(cmd_line, background=True, fail=False)
1038    # Wait for the verifier to start
1039    while sim.dfs_num_bound_progs() <= 2:
1040        pass
1041    sim.remove()
1042    end = time.time()
1043    ret, _ = cmd_result(tc_proc, fail=False)
1044    time_diff = end - start
1045    log("Time", "start:\t%s\nend:\t%s\ndiff:\t%s" % (start, end, time_diff))
1046
1047    fail(ret == 0, "Managed to load TC filter on a unregistering device")
1048    delay_sec = delay_msec * 0.001
1049    fail(time_diff < delay_sec, "Removal process took %s, expected %s" %
1050         (time_diff, delay_sec))
1051
1052    # Remove all pinned files and reinstantiate the netdev
1053    clean_up()
1054    bpftool_prog_list_wait(expected=0)
1055
1056    sim = NetdevSim()
1057    map_obj = bpf_obj("sample_map_ret0.o")
1058    start_test("Test loading program with maps...")
1059    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1060
1061    start_test("Test bpftool bound info reporting (own ns)...")
1062    check_dev_info(False, "")
1063
1064    start_test("Test bpftool bound info reporting (other ns)...")
1065    ns = mknetns()
1066    sim.set_ns(ns)
1067    check_dev_info(True, "")
1068
1069    start_test("Test bpftool bound info reporting (remote ns)...")
1070    check_dev_info(False, ns)
1071
1072    start_test("Test bpftool bound info reporting (back to own ns)...")
1073    sim.set_ns("")
1074    check_dev_info(False, "")
1075
1076    prog_file, _ = pin_prog("/sys/fs/bpf/tmp_prog")
1077    map_file, _ = pin_map("/sys/fs/bpf/tmp_map", idx=1, expected=2)
1078    sim.remove()
1079
1080    start_test("Test bpftool bound info reporting (removed dev)...")
1081    check_dev_info_removed(prog_file=prog_file, map_file=map_file)
1082
1083    # Remove all pinned files and reinstantiate the netdev
1084    clean_up()
1085    bpftool_prog_list_wait(expected=0)
1086
1087    sim = NetdevSim()
1088
1089    start_test("Test map update (no flags)...")
1090    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1091    maps = bpftool_map_list(expected=2)
1092    array = maps[0] if maps[0]["type"] == "array" else maps[1]
1093    htab = maps[0] if maps[0]["type"] == "hash" else maps[1]
1094    for m in maps:
1095        for i in range(2):
1096            bpftool("map update id %d key %s value %s" %
1097                    (m["id"], int2str("I", i), int2str("Q", i * 3)))
1098
1099    for m in maps:
1100        ret, _ = bpftool("map update id %d key %s value %s" %
1101                         (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1102                         fail=False)
1103        fail(ret == 0, "added too many entries")
1104
1105    start_test("Test map update (exists)...")
1106    for m in maps:
1107        for i in range(2):
1108            bpftool("map update id %d key %s value %s exist" %
1109                    (m["id"], int2str("I", i), int2str("Q", i * 3)))
1110
1111    for m in maps:
1112        ret, err = bpftool("map update id %d key %s value %s exist" %
1113                           (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1114                           fail=False)
1115        fail(ret == 0, "updated non-existing key")
1116        fail(err["error"].find("No such file or directory") == -1,
1117             "expected ENOENT, error is '%s'" % (err["error"]))
1118
1119    start_test("Test map update (noexist)...")
1120    for m in maps:
1121        for i in range(2):
1122            ret, err = bpftool("map update id %d key %s value %s noexist" %
1123                               (m["id"], int2str("I", i), int2str("Q", i * 3)),
1124                               fail=False)
1125        fail(ret == 0, "updated existing key")
1126        fail(err["error"].find("File exists") == -1,
1127             "expected EEXIST, error is '%s'" % (err["error"]))
1128
1129    start_test("Test map dump...")
1130    for m in maps:
1131        _, entries = bpftool("map dump id %d" % (m["id"]))
1132        for i in range(2):
1133            key = str2int(entries[i]["key"])
1134            fail(key != i, "expected key %d, got %d" % (key, i))
1135            val = str2int(entries[i]["value"])
1136            fail(val != i * 3, "expected value %d, got %d" % (val, i * 3))
1137
1138    start_test("Test map getnext...")
1139    for m in maps:
1140        _, entry = bpftool("map getnext id %d" % (m["id"]))
1141        key = str2int(entry["next_key"])
1142        fail(key != 0, "next key %d, expected %d" % (key, 0))
1143        _, entry = bpftool("map getnext id %d key %s" %
1144                           (m["id"], int2str("I", 0)))
1145        key = str2int(entry["next_key"])
1146        fail(key != 1, "next key %d, expected %d" % (key, 1))
1147        ret, err = bpftool("map getnext id %d key %s" %
1148                           (m["id"], int2str("I", 1)), fail=False)
1149        fail(ret == 0, "got next key past the end of map")
1150        fail(err["error"].find("No such file or directory") == -1,
1151             "expected ENOENT, error is '%s'" % (err["error"]))
1152
1153    start_test("Test map delete (htab)...")
1154    for i in range(2):
1155        bpftool("map delete id %d key %s" % (htab["id"], int2str("I", i)))
1156
1157    start_test("Test map delete (array)...")
1158    for i in range(2):
1159        ret, err = bpftool("map delete id %d key %s" %
1160                           (htab["id"], int2str("I", i)), fail=False)
1161        fail(ret == 0, "removed entry from an array")
1162        fail(err["error"].find("No such file or directory") == -1,
1163             "expected ENOENT, error is '%s'" % (err["error"]))
1164
1165    start_test("Test map remove...")
1166    sim.unset_xdp("offload")
1167    bpftool_map_list_wait(expected=0)
1168    sim.remove()
1169
1170    sim = NetdevSim()
1171    sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1172    sim.remove()
1173    bpftool_map_list_wait(expected=0)
1174
1175    start_test("Test map creation fail path...")
1176    sim = NetdevSim()
1177    sim.dfs["bpf_map_accept"] = "N"
1178    ret, _ = sim.set_xdp(map_obj, "offload", JSON=False, fail=False)
1179    fail(ret == 0,
1180         "netdevsim didn't refuse to create a map with offload disabled")
1181
1182    sim.remove()
1183
1184    start_test("Test multi-dev ASIC program reuse...")
1185    simA = NetdevSim()
1186    simB1 = NetdevSim()
1187    simB2 = NetdevSim(link=simB1)
1188    simB3 = NetdevSim(link=simB1)
1189    sims = (simA, simB1, simB2, simB3)
1190    simB = (simB1, simB2, simB3)
1191
1192    bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA",
1193                      dev=simA['ifname'])
1194    progA = bpf_pinned("/sys/fs/bpf/nsimA")
1195    bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB",
1196                      dev=simB1['ifname'])
1197    progB = bpf_pinned("/sys/fs/bpf/nsimB")
1198
1199    simA.set_xdp(progA, "offload", JSON=False)
1200    for d in simB:
1201        d.set_xdp(progB, "offload", JSON=False)
1202
1203    start_test("Test multi-dev ASIC cross-dev replace...")
1204    ret, _ = simA.set_xdp(progB, "offload", force=True, JSON=False, fail=False)
1205    fail(ret == 0, "cross-ASIC program allowed")
1206    for d in simB:
1207        ret, _ = d.set_xdp(progA, "offload", force=True, JSON=False, fail=False)
1208        fail(ret == 0, "cross-ASIC program allowed")
1209
1210    start_test("Test multi-dev ASIC cross-dev install...")
1211    for d in sims:
1212        d.unset_xdp("offload")
1213
1214    ret, _, err = simA.set_xdp(progB, "offload", force=True, JSON=False,
1215                               fail=False, include_stderr=True)
1216    fail(ret == 0, "cross-ASIC program allowed")
1217    check_extack_nsim(err, "program bound to different dev.", args)
1218    for d in simB:
1219        ret, _, err = d.set_xdp(progA, "offload", force=True, JSON=False,
1220                                fail=False, include_stderr=True)
1221        fail(ret == 0, "cross-ASIC program allowed")
1222        check_extack_nsim(err, "program bound to different dev.", args)
1223
1224    start_test("Test multi-dev ASIC cross-dev map reuse...")
1225
1226    mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
1227    mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
1228
1229    ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
1230                               dev=simB3['ifname'],
1231                               maps=["idx 0 id %d" % (mapB)],
1232                               fail=False)
1233    fail(ret != 0, "couldn't reuse a map on the same ASIC")
1234    rm("/sys/fs/bpf/nsimB_")
1235
1236    ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_",
1237                                    dev=simA['ifname'],
1238                                    maps=["idx 0 id %d" % (mapB)],
1239                                    fail=False, include_stderr=True)
1240    fail(ret == 0, "could reuse a map on a different ASIC")
1241    fail(err.count("offload device mismatch between prog and map") == 0,
1242         "error message missing for cross-ASIC map")
1243
1244    ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
1245                                    dev=simB1['ifname'],
1246                                    maps=["idx 0 id %d" % (mapA)],
1247                                    fail=False, include_stderr=True)
1248    fail(ret == 0, "could reuse a map on a different ASIC")
1249    fail(err.count("offload device mismatch between prog and map") == 0,
1250         "error message missing for cross-ASIC map")
1251
1252    start_test("Test multi-dev ASIC cross-dev destruction...")
1253    bpftool_prog_list_wait(expected=2)
1254
1255    simA.remove()
1256    bpftool_prog_list_wait(expected=1)
1257
1258    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1259    fail(ifnameB != simB1['ifname'], "program not bound to originial device")
1260    simB1.remove()
1261    bpftool_prog_list_wait(expected=1)
1262
1263    start_test("Test multi-dev ASIC cross-dev destruction - move...")
1264    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1265    fail(ifnameB not in (simB2['ifname'], simB3['ifname']),
1266         "program not bound to remaining devices")
1267
1268    simB2.remove()
1269    ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
1270    fail(ifnameB != simB3['ifname'], "program not bound to remaining device")
1271
1272    simB3.remove()
1273    bpftool_prog_list_wait(expected=0)
1274
1275    start_test("Test multi-dev ASIC cross-dev destruction - orphaned...")
1276    ret, out = bpftool("prog show %s" % (progB), fail=False)
1277    fail(ret == 0, "got information about orphaned program")
1278    fail("error" not in out, "no error reported for get info on orphaned")
1279    fail(out["error"] != "can't get prog info: No such device",
1280         "wrong error for get info on orphaned")
1281
1282    print("%s: OK" % (os.path.basename(__file__)))
1283
1284finally:
1285    log("Clean up...", "", level=1)
1286    log_level_inc()
1287    clean_up()
1288