1#!/usr/bin/env python3
2# Copyright (c) 2023 Intel Corporation
3#
4# SPDX-License-Identifier: Apache-2.0
5"""
6Tests for jobserver.py classes' methods
7"""
8
9import functools
10import mock
11import os
12import pytest
13import sys
14
15from contextlib import nullcontext
16from errno import ENOENT
17from selectors import EVENT_READ
18
19# Job server only works on Linux for now.
20pytestmark = pytest.mark.skipif(sys.platform != 'linux', reason='JobServer only works on Linux.')
21if sys.platform == 'linux':
22    from twisterlib.jobserver import GNUMakeJobClient, GNUMakeJobServer, JobClient, JobHandle
23    from fcntl import F_GETFL
24
25
26def test_jobhandle(capfd):
27    def f(a, b, c=None, d=None):
28        print(f'{a}, {b}, {c}, {d}')
29
30    def exiter():
31        with JobHandle(f, 1, 2, c='three', d=4):
32            return
33
34    exiter()
35
36    out, err = capfd.readouterr()
37    sys.stdout.write(out)
38    sys.stderr.write(err)
39
40    assert '1, 2, three, 4' in out
41
42
43def test_jobclient_get_job():
44    jc = JobClient()
45
46    job = jc.get_job()
47
48    assert isinstance(job, JobHandle)
49    assert job.release_func is None
50
51
52def test_jobclient_env():
53    env = JobClient.env()
54
55    assert env == {}
56
57
58def test_jobclient_pass_fds():
59    fds = JobClient.pass_fds()
60
61    assert fds == []
62
63
64TESTDATA_1 = [
65    ({}, {'env': {'k': 'v'}, 'pass_fds': []}),
66    ({'env': {}, 'pass_fds': ['fd']}, {'env': {}, 'pass_fds': ['fd']}),
67]
68
69@pytest.mark.parametrize(
70    'kwargs, expected_kwargs',
71    TESTDATA_1,
72    ids=['no values', 'preexisting values']
73)
74def test_jobclient_popen(kwargs, expected_kwargs):
75    jc = JobClient()
76
77    argv = ['cmd', 'and', 'some', 'args']
78    proc_mock = mock.Mock()
79    popen_mock = mock.Mock(return_value=proc_mock)
80    env_mock = {'k': 'v'}
81
82    with mock.patch('subprocess.Popen', popen_mock), \
83         mock.patch('os.environ', env_mock):
84        proc = jc.popen(argv, **kwargs)
85
86    popen_mock.assert_called_once_with(argv, **expected_kwargs)
87    assert proc == proc_mock
88
89
90TESTDATA_2 = [
91    (False, 0),
92    (True, 0),
93    (False, 4),
94    (True, 16),
95]
96
97@pytest.mark.parametrize(
98    'inheritable, internal_jobs',
99    TESTDATA_2,
100    ids=['no inheritable, no internal', 'inheritable, no internal',
101         'no inheritable, internal', 'inheritable, internal']
102)
103def test_gnumakejobclient_dunders(inheritable, internal_jobs):
104    inherit_read_fd = mock.Mock()
105    inherit_write_fd = mock.Mock()
106    inheritable_pipe = (inherit_read_fd, inherit_write_fd) if inheritable else \
107                      None
108
109    internal_read_fd = mock.Mock()
110    internal_write_fd = mock.Mock()
111
112    def mock_pipe():
113        return (internal_read_fd, internal_write_fd)
114
115    close_mock = mock.Mock()
116    write_mock = mock.Mock()
117    set_blocking_mock = mock.Mock()
118    selector_mock = mock.Mock()
119
120    def deleter():
121        jobs = mock.Mock()
122        makeflags = mock.Mock()
123
124        gmjc = GNUMakeJobClient(
125            inheritable_pipe,
126            jobs,
127            internal_jobs=internal_jobs,
128            makeflags=makeflags
129        )
130
131        assert gmjc.jobs == jobs
132        if internal_jobs:
133            write_mock.assert_called_once_with(internal_write_fd,
134                                               b'+' * internal_jobs)
135            set_blocking_mock.assert_any_call(internal_read_fd, False)
136            selector_mock().register.assert_any_call(internal_read_fd,
137                                                     EVENT_READ,
138                                                     internal_write_fd)
139        if inheritable:
140            set_blocking_mock.assert_any_call(inherit_read_fd, False)
141            selector_mock().register.assert_any_call(inherit_read_fd,
142                                                     EVENT_READ,
143                                                     inherit_write_fd)
144
145    with mock.patch('os.close', close_mock), \
146         mock.patch('os.write', write_mock), \
147         mock.patch('os.set_blocking', set_blocking_mock), \
148         mock.patch('os.pipe', mock_pipe), \
149         mock.patch('selectors.DefaultSelector', selector_mock):
150        deleter()
151
152    if internal_jobs:
153        close_mock.assert_any_call(internal_read_fd)
154        close_mock.assert_any_call(internal_write_fd)
155    if inheritable:
156        close_mock.assert_any_call(inherit_read_fd)
157        close_mock.assert_any_call(inherit_write_fd)
158
159
160TESTDATA_3 = [
161    (
162        {'MAKEFLAGS': '-j1'},
163        0,
164        (False, False),
165        ['Running in sequential mode (-j1)'],
166        None,
167        [None, 1],
168        {'internal_jobs': 1, 'makeflags': '-j1'}
169    ),
170    (
171        {'MAKEFLAGS': 'n--jobserver-auth=0,1'},
172        1,
173        (True, True),
174        [
175            '-jN forced on command line; ignoring GNU make jobserver',
176            'MAKEFLAGS contained dry-run flag'
177        ],
178        0,
179        None,
180        None
181    ),
182    (
183        {'MAKEFLAGS': '--jobserver-auth=0,1'},
184        0,
185        (True, True),
186        ['using GNU make jobserver'],
187        None,
188        [[0, 1], 0],
189        {'internal_jobs': 1, 'makeflags': '--jobserver-auth=0,1'}
190    ),
191    (
192        {'MAKEFLAGS': '--jobserver-auth=123,321'},
193        0,
194        (False, False),
195        ['No file descriptors; ignoring GNU make jobserver'],
196        None,
197        [None, 0],
198        {'internal_jobs': 1, 'makeflags': '--jobserver-auth=123,321'}
199    ),
200    (
201        {'MAKEFLAGS': '--jobserver-auth=0,1'},
202        0,
203        (False, True),
204        [f'FD 0 is not readable (flags=2); ignoring GNU make jobserver'],
205        None,
206        [None, 0],
207        {'internal_jobs': 1, 'makeflags': '--jobserver-auth=0,1'}
208    ),
209    (
210        {'MAKEFLAGS': '--jobserver-auth=0,1'},
211        0,
212        (True, False),
213        [f'FD 1 is not writable (flags=2); ignoring GNU make jobserver'],
214        None,
215        [None, 0],
216        {'internal_jobs': 1, 'makeflags': '--jobserver-auth=0,1'}
217    ),
218    (None, 0, (False, False), [], None, None, None),
219]
220
221@pytest.mark.parametrize(
222    'env, jobs, fcntl_ok_per_pipe, expected_logs,' \
223    ' exit_code, expected_args, expected_kwargs',
224    TESTDATA_3,
225    ids=['env, no jobserver-auth', 'env, jobs, dry run', 'env, no jobs',
226         'env, no jobs, oserror', 'env, no jobs, wrong read pipe',
227         'env, no jobs, wrong write pipe', 'environ, no makeflags']
228)
229def test_gnumakejobclient_from_environ(
230    caplog,
231    env,
232    jobs,
233    fcntl_ok_per_pipe,
234    expected_logs,
235    exit_code,
236    expected_args,
237    expected_kwargs
238):
239    def mock_fcntl(fd, flag):
240        if flag == F_GETFL:
241            if fd == 0:
242                if fcntl_ok_per_pipe[0]:
243                    return os.O_RDONLY
244                else:
245                    return 2
246            elif fd == 1:
247                if fcntl_ok_per_pipe[1]:
248                    return os.O_WRONLY
249                else:
250                    return 2
251        raise OSError(ENOENT, 'dummy error')
252
253    gmjc_init_mock = mock.Mock(return_value=None)
254
255    with mock.patch('fcntl.fcntl', mock_fcntl), \
256         mock.patch('os.close', mock.Mock()), \
257         mock.patch('twisterlib.jobserver.GNUMakeJobClient.__init__',
258                    gmjc_init_mock), \
259         pytest.raises(SystemExit) if exit_code is not None else \
260         nullcontext() as se:
261        gmjc = GNUMakeJobClient.from_environ(env=env, jobs=jobs)
262
263        # As patching __del__ is hard to do, we'll instead
264        # cover possible exceptions and mock os calls
265        if gmjc:
266            gmjc._inheritable_pipe = getattr(gmjc, '_inheritable_pipe', None)
267        if gmjc:
268            gmjc._internal_pipe = getattr(gmjc, '_internal_pipe', None)
269
270    assert all([log in caplog.text for log in expected_logs])
271
272    if se:
273        assert str(se.value) == str(exit_code)
274        return
275
276    if expected_args is None and expected_kwargs is None:
277        assert gmjc is None
278    else:
279        gmjc_init_mock.assert_called_once_with(*expected_args,
280                                               **expected_kwargs)
281
282
283
284def test_gnumakejobclient_get_job():
285    inherit_read_fd = mock.Mock()
286    inherit_write_fd = mock.Mock()
287    inheritable_pipe = (inherit_read_fd, inherit_write_fd)
288
289    internal_read_fd = mock.Mock()
290    internal_write_fd = mock.Mock()
291
292    def mock_pipe():
293        return (internal_read_fd, internal_write_fd)
294
295    selected = [[mock.Mock(fd=0, data=1)], [mock.Mock(fd=1, data=0)]]
296
297    def mock_select():
298        nonlocal selected
299        return selected
300
301    def mock_read(fd, length):
302        nonlocal selected
303        if fd == 0:
304            selected = selected[1:]
305            raise BlockingIOError
306        return b'?' * length
307
308    close_mock = mock.Mock()
309    write_mock = mock.Mock()
310    set_blocking_mock = mock.Mock()
311    selector_mock = mock.Mock()
312    selector_mock().select = mock.Mock(side_effect=mock_select)
313
314    def deleter():
315        jobs = mock.Mock()
316
317        gmjc = GNUMakeJobClient(
318            inheritable_pipe,
319            jobs
320        )
321
322        with mock.patch('os.read', side_effect=mock_read):
323            job = gmjc.get_job()
324            with job:
325                expected_func = functools.partial(os.write, 0, b'?')
326
327                assert job.release_func.func == expected_func.func
328                assert job.release_func.args == expected_func.args
329                assert job.release_func.keywords == expected_func.keywords
330
331    with mock.patch('os.close', close_mock), \
332         mock.patch('os.write', write_mock), \
333         mock.patch('os.set_blocking', set_blocking_mock), \
334         mock.patch('os.pipe', mock_pipe), \
335         mock.patch('selectors.DefaultSelector', selector_mock):
336        deleter()
337
338    write_mock.assert_any_call(0, b'?')
339
340
341TESTDATA_4 = [
342    ('dummy makeflags', mock.ANY, mock.ANY, {'MAKEFLAGS': 'dummy makeflags'}),
343    (None, 0, False, {'MAKEFLAGS': ''}),
344    (None, 1, True, {'MAKEFLAGS': ' -j1'}),
345    (None, 2, True, {'MAKEFLAGS': ' -j2 --jobserver-auth=0,1'}),
346    (None, 0, True, {'MAKEFLAGS': ' --jobserver-auth=0,1'}),
347]
348
349@pytest.mark.parametrize(
350    'makeflags, jobs, use_inheritable_pipe, expected_makeflags',
351    TESTDATA_4,
352    ids=['preexisting makeflags', 'no jobs, no pipe', 'one job',
353         ' multiple jobs', 'no jobs']
354)
355def test_gnumakejobclient_env(
356    makeflags,
357    jobs,
358    use_inheritable_pipe,
359    expected_makeflags
360):
361    inheritable_pipe = (0, 1) if use_inheritable_pipe else None
362
363    selector_mock = mock.Mock()
364
365    env = None
366
367    def deleter():
368        gmjc = GNUMakeJobClient(None, None)
369        gmjc.jobs = jobs
370        gmjc._makeflags = makeflags
371        gmjc._inheritable_pipe = inheritable_pipe
372
373        nonlocal env
374        env = gmjc.env()
375
376    with mock.patch.object(GNUMakeJobClient, '__del__', mock.Mock()), \
377         mock.patch('selectors.DefaultSelector', selector_mock):
378        deleter()
379
380    assert env == expected_makeflags
381
382
383TESTDATA_5 = [
384    (2, False, []),
385    (1, True, []),
386    (2, True, (0, 1)),
387    (0, True, (0, 1)),
388]
389
390@pytest.mark.parametrize(
391    'jobs, use_inheritable_pipe, expected_fds',
392    TESTDATA_5,
393    ids=['no pipe', 'one job', ' multiple jobs', 'no jobs']
394)
395def test_gnumakejobclient_pass_fds(jobs, use_inheritable_pipe, expected_fds):
396    inheritable_pipe = (0, 1) if use_inheritable_pipe else None
397
398    selector_mock = mock.Mock()
399
400    fds = None
401
402    def deleter():
403        gmjc = GNUMakeJobClient(None, None)
404        gmjc.jobs = jobs
405        gmjc._inheritable_pipe = inheritable_pipe
406
407        nonlocal fds
408        fds = gmjc.pass_fds()
409
410    with mock.patch('twisterlib.jobserver.GNUMakeJobClient.__del__',
411                    mock.Mock()), \
412         mock.patch('selectors.DefaultSelector', selector_mock):
413        deleter()
414
415    assert fds == expected_fds
416
417
418TESTDATA_6 = [
419    (0, 8),
420    (32, 16),
421    (4, 4),
422]
423
424@pytest.mark.parametrize(
425    'jobs, expected_jobs',
426    TESTDATA_6,
427    ids=['no jobs', 'too many jobs', 'valid jobs']
428)
429def test_gnumakejobserver(jobs, expected_jobs):
430    def mock_init(self, p, j):
431        self._inheritable_pipe = p
432        self._internal_pipe = None
433        self.jobs = j
434
435    pipe = (0, 1)
436    cpu_count = 8
437    pipe_buf = 16
438
439    selector_mock = mock.Mock()
440    write_mock = mock.Mock()
441    del_mock = mock.Mock()
442
443    def deleter():
444        GNUMakeJobServer(jobs=jobs)
445
446    with mock.patch.object(GNUMakeJobClient, '__del__', del_mock), \
447         mock.patch.object(GNUMakeJobClient, '__init__', mock_init), \
448         mock.patch('os.pipe', return_value=pipe), \
449         mock.patch('os.write', write_mock), \
450         mock.patch('multiprocessing.cpu_count', return_value=cpu_count), \
451         mock.patch('select.PIPE_BUF', pipe_buf), \
452         mock.patch('selectors.DefaultSelector', selector_mock):
453        deleter()
454
455    write_mock.assert_called_once_with(pipe[1], b'+' * expected_jobs)
456