-
-
Notifications
You must be signed in to change notification settings - Fork 189
/
runTests.py
executable file
·453 lines (381 loc) · 13.8 KB
/
runTests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
#!/usr/bin/env python3
"""
Replacement for runtest target in Makefile.
Call script with '-h' as an option to see a helpful message.
"""
from __future__ import print_function
import glob
import os
import os.path
import platform
import re
import subprocess
import sys
import time
from argparse import ArgumentParser, RawTextHelpFormatter
winsfx = ".exe"
testsfx = "_test.cpp"
allowed_paths_with_jumbo = [
"test/unit/math/prim/",
"test/unit/math/prim/constraint",
"test/unit/math/",
"test/unit/math/rev/",
"test/unit/math/rev/constraint",
"test/unit/math/fwd/",
"test/unit/math/fwd/constraint",
"test/unit/math/mix/",
"test/unit/math/mix/fun/",
"test/unit/math/opencl/",
"test/unit/",
]
jumbo_folders = [
"test/unit/math/prim/constraint",
"test/unit/math/prim/core",
"test/unit/math/prim/err",
"test/unit/math/prim/fun",
"test/unit/math/prim/functor",
"test/unit/math/prim/meta",
"test/unit/math/prim/prob",
"test/unit/math/rev/constraint",
"test/unit/math/rev/core",
"test/unit/math/rev/err",
"test/unit/math/rev/fun",
"test/unit/math/rev/functor",
"test/unit/math/rev/meta",
"test/unit/math/rev/prob",
"test/unit/math/fwd/constraint",
"test/unit/math/fwd/core",
"test/unit/math/fwd/fun",
"test/unit/math/fwd/functor",
"test/unit/math/fwd/meta",
"test/unit/math/fwd/prob",
"test/unit/math/mix/core",
"test/unit/math/mix/fun",
"test/unit/math/mix/functor",
"test/unit/math/mix/meta",
"test/unit/math/mix/prob",
"test/unit/math/opencl/device_functions",
"test/unit/math/opencl/kernel_generator",
"test/unit/math/opencl/prim",
"test/unit/math/opencl/prim/constraint",
"test/unit/math/opencl/rev",
"test/unit/math/opencl/rev/constraint",
]
def processCLIArgs():
"""
Define and process the command line interface to the runTests.py script.
"""
cli_description = "Generate and run stan math library tests."
cli_epilog = "See more information at: https://github.com/stan-dev/math"
parser = ArgumentParser(
description=cli_description,
epilog=cli_epilog,
formatter_class=RawTextHelpFormatter,
)
# Now define all the rules of the command line args and opts
parser.add_argument(
"-j", metavar="N", type=int, default=1, help="number of cores for make to use"
)
parser.add_argument(
"-e",
metavar="M",
type=int,
default=-1,
help="number of files to split expressions tests in",
)
f_help_msg = "Only tests with file names matching these will be executed.\n"
f_help_msg += "Example: '-f chol', '-f opencl', '-f prim'"
parser.add_argument("-f", type=str, default=[], action="append", help=f_help_msg)
changed_help = (
"Use git to determine which tests may have changed, and run only those"
)
parser.add_argument("--changed", action="store_true", help=changed_help)
parser.add_argument(
"-d",
"--debug",
dest="debug",
action="store_true",
help="request additional script debugging output.",
)
parser.add_argument(
"-m",
"--make-only",
dest="make_only",
action="store_true",
help="Don't run tests, just try to make them.",
)
parser.add_argument(
"--run-all",
dest="run_all",
action="store_true",
help="Don't stop at the first test failure, run all of them.",
)
parser.add_argument(
"--only-functions",
nargs="+",
type=str,
default=[],
help="Function names to run expression tests for. Default: all functions",
)
parser.add_argument(
"--jumbo-test",
dest="do_jumbo",
action="store_true",
help="Build/run jumbo tests.",
)
tests_help_msg = "The path(s) to the test case(s) to run.\n"
tests_help_msg += "Example: 'test/unit', 'test/prob', and/or\n"
tests_help_msg += " 'test/unit/math/prim/fun/abs_test.cpp'"
parser.add_argument("tests", nargs="*", type=str, help=tests_help_msg)
# And parse the command line against those rules
return parser.parse_args()
def stopErr(msg, returncode):
"""Report an error message to stderr and exit with a given code."""
sys.stderr.write("%s\n" % msg)
sys.stderr.write("exit now (%s)\n" % time.strftime("%x %X %Z"))
sys.exit(returncode)
def isWin():
return platform.system().lower().startswith(
"windows"
) or os.name.lower().startswith("windows")
batchSize = 20 if isWin() else 200
jumboSize = 5 if isWin() else 12
maxChangedTests = 20
def mungeName(name):
"""Set up the makefile target name"""
if name.endswith(testsfx):
name = name.replace(testsfx, "_test")
if isWin():
name += winsfx
name = name.replace("\\", "/")
return name
def doCommand(command, exit_on_failure=True):
"""Run command as a shell command and report/exit on errors."""
print("------------------------------------------------------------")
print("%s" % command)
p1 = subprocess.Popen(command, shell=True)
p1.wait()
if exit_on_failure and (not (p1.returncode is None) and not (p1.returncode == 0)):
stopErr("%s failed" % command, p1.returncode)
def generateTests(j):
"""Generate all tests and pass along the j parameter to make."""
doCommand("make -j%d generate-tests -s" % (j or 1))
def divide_chunks(l, n):
# looping till length l
for i in range(0, len(l), n):
yield l[i : i + n]
include_pattern = re.compile(r'^\s*#include\s+<.*>$')
def generateJumboTests(paths, debug=False):
jumbo_files_to_create = []
jumbo_files = []
for p in paths:
if not p.endswith(testsfx) and not p.endswith("/"):
p = p + "/"
if p in allowed_paths_with_jumbo:
jumbo_files_to_create.extend([x for x in jumbo_folders if x.startswith(p)])
else:
stopErr("The --jumbo flag is only allowed with top level folders.", 10)
for jf in jumbo_files_to_create:
tests_in_subfolder = sorted(os.path.join(jf, x) for x in os.listdir(jf) if x.endswith(testsfx))
for i, tests in enumerate(divide_chunks(tests_in_subfolder, jumboSize)):
includes = []
jumbo_contents = []
for t in tests:
with open(t, "r") as test_file:
contents_raw = test_file.read()
test_contents = []
for l in contents_raw.splitlines():
if include_pattern.fullmatch(l.strip()):
includes.append(l.strip())
continue
test_contents.append(l)
jumbo_contents.append('\n'.join(test_contents))
jumbo_file_path = jf + "_" + str(i) + testsfx
jumbo_files.append(jumbo_file_path)
if debug:
print("Generating jumbo test file '{}' with tests: {}".format( jumbo_file_path, ', '.join(tests)))
with open(jumbo_file_path, "w") as jumbo_file:
jumbo_file.write('\n'.join(list(dict.fromkeys(includes))))
jumbo_file.write('\n')
jumbo_file.write('\n'.join(jumbo_contents))
return jumbo_files
def cleanupJumboTests(paths):
for f in paths:
if os.path.exists(f):
os.remove(f)
def makeTest(name, j):
"""Run the make command for a given single test."""
doCommand("make -j%d %s" % (j or 1, name))
def commandExists(command):
p = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
p.wait()
return p.returncode != 127
def runTest(name, run_all=False, mpi=False, j=1):
executable = mungeName(name).replace("/", os.sep)
xml = mungeName(name).replace(winsfx, "")
command = '%s --gtest_output="xml:%s.xml"' % (executable, xml)
if mpi:
if not commandExists("mpirun"):
stopErr(
"Error: need to have mpi (and mpirun) installed to run mpi tests",
-1,
)
if "mpi_" in name:
j = j > 2 and j or 2
else:
j = 1
command = "mpirun -np {} {}".format(j, command)
doCommand(command, not run_all)
def test_files_in_folder(folder):
"""Returns a list of test files (*_test.cpp) in the folder and all
its subfolders recursively. The folder can be written with
wildcards as with the Unix find command.
"""
files = []
for f in glob.glob(folder):
if os.path.isdir(f):
files.extend(test_files_in_folder(f + os.sep + "**"))
else:
if f.endswith(testsfx):
files.append(f)
return files
def findTests(base_path, filter_names, do_jumbo=False):
tests = []
for path in base_path:
if (not os.path.isdir(path)) and path.endswith("_test"):
tests.append(path + ".cpp")
else:
tests.extend(test_files_in_folder(path))
tests = map(mungeName, tests)
tests = [
test
for test in tests
if all(filter_name in test for filter_name in filter_names)
]
if do_jumbo:
filtered_jumbo_tests = []
for t in tests:
add = True
for k in jumbo_folders:
k = k + "/"
if t.startswith(k):
add = False
break
if add:
filtered_jumbo_tests.append(t)
tests = filtered_jumbo_tests
return tests
def findChangedTests(debug):
import subprocess
changed_files = subprocess.run(
["git", "diff", "--name-only", "--diff-filter=d", "origin/develop...HEAD"],
text=True, capture_output=True
).stdout.splitlines()
if debug:
print("Changed files:", changed_files)
# changes in prim should also test other non-prim signatures
test_deps = {"prim/": ["prim/", "rev/", "mix/", "fwd/"], "rev/": ["rev/", "mix/"], "fwd/": ["fwd/", "mix/"]}
changed_tests = set()
for f in changed_files:
if 'stan/' in f and f.endswith('.hpp') and 'opencl' not in f: # changed fn
maybe_test = f.replace('stan/', 'test/unit/').replace('.hpp', testsfx)
for (path, replacements) in test_deps.items():
if path in maybe_test:
for rep in replacements:
maybe_test = maybe_test.replace(path, rep)
if os.path.exists(maybe_test):
changed_tests.add(maybe_test)
if f.endswith(testsfx) and 'opencl' not in f:
changed_tests.add(f)
if len(changed_tests) > maxChangedTests:
stopErr("Number of changed tests excluded maximum, not running priority tests", 0)
return list(map(mungeName, changed_tests))
def batched(tests):
return [tests[i : i + batchSize] for i in range(0, len(tests), batchSize)]
def handleExpressionTests(tests, only_functions, n_test_files):
expression_tests = False
for n, i in list(enumerate(tests))[::-1]:
if "test/expressions" in i or "test\\expressions" in i:
del tests[n]
expression_tests = True
if expression_tests:
HERE = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(HERE, "test"))
sys.path.append(os.path.join(HERE, "test/expressions"))
import generate_expression_tests
generate_expression_tests.main(only_functions, n_test_files)
for i in range(n_test_files):
tests.append("test/expressions/tests%d_test.cpp" % i)
elif only_functions:
stopErr(
"--only-functions can only be specified if running expression tests (test/expressions)",
-1,
)
def checkToolchainPathWindows():
if isWin():
p1 = subprocess.Popen(
"where.exe make",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
)
out, err = p1.communicate()
if re.search(r" |\(|\)", out):
stopErr(
"The RTools toolchain is installed in a path with spaces or bracket. Please reinstall to a valid path.",
-1,
)
def main():
inputs = processCLIArgs()
checkToolchainPathWindows()
try:
with open("make/local") as f:
stan_mpi = "STAN_MPI" in f.read()
except IOError:
stan_mpi = False
jumboFiles = []
if inputs.changed:
tests = findChangedTests(inputs.debug)
else:
# pass 0: generate all auto-generated tests
if any("test/prob" in arg for arg in inputs.tests):
generateTests(inputs.j)
if inputs.do_jumbo:
jumboFiles = generateJumboTests(inputs.tests)
if inputs.e == -1:
if inputs.j == 1:
num_expr_test_files = 1
else:
num_expr_test_files = inputs.j * 4
else:
num_expr_test_files = inputs.e
handleExpressionTests(inputs.tests, inputs.only_functions, num_expr_test_files)
tests = findTests(inputs.tests, inputs.f, inputs.do_jumbo)
if not tests:
if inputs.changed:
stopErr("No changed tests were found!", 0)
stopErr(
"No matching tests found. Check the path passed on the command line", -1
)
try:
# pass 1: make test executables
for batch in batched(tests):
if inputs.debug:
print("Test batch: ", batch)
makeTest(" ".join(batch), inputs.j)
if not inputs.make_only:
# pass 2: run test targets
for t in tests:
if inputs.debug:
print("run single test: %s" % t)
runTest(t, inputs.run_all, mpi=stan_mpi, j=inputs.j)
except BaseException as e:
print(e, file=sys.stderr)
sys.exit(1)
finally:
cleanupJumboTests(jumboFiles)
pass
if __name__ == "__main__":
main()