forked from ivy-llc/ivy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
determine_test_coverage.py
118 lines (107 loc) · 4.09 KB
/
determine_test_coverage.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
import os
import sys
from pydriller import Repository
import pickle # noqa
from tqdm import tqdm
import bz2
import _pickle as cPickle
# Shared Map
tests = {}
os.system("git config --global --add safe.directory /ivy")
N = 4
run_iter = int(sys.argv[1])
os.system(
"pytest --disable-pytest-warnings ivy_tests/test_ivy --my_test_dump true > test_names" # noqa
)
test_names = []
with open("test_names") as f:
i = 0
for line in f:
i += 1
if i <= 5:
continue
test_names.append(line[:-1])
test_names = test_names[:-3]
# Create a Dictionary of Test Names to Index
tests["index_mapping"] = test_names
tests["tests_mapping"] = {}
for i in range(len(test_names)):
tests["tests_mapping"][test_names[i]] = i
directories = [
"ivy",
"ivy/array",
"ivy/container",
"ivy/functional",
"ivy/functional/backends",
"ivy/functional/backends/jax",
"ivy/functional/backends/numpy",
"ivy/functional/backends/torch",
"ivy/functional/backends/tensorflow",
"ivy/functional/frontends",
"ivy/functional/frontends/jax",
"ivy/functional/frontends/numpy",
"ivy/functional/frontends/torch",
"ivy/functional/frontends/tensorflow",
"ivy/functional/ivy",
"ivy/stateful",
"ivy_tests",
"ivy_tests/test_ivy",
"ivy_tests/test_ivy/test_frontends",
"ivy_tests/test_ivy/test_frontends/test_jax",
"ivy_tests/test_ivy/test_frontends/test_numpy",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_creation_routines",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_fft",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_indexing_routines",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_linear_algebra",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_logic",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_ma",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_manipulation_routines",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_matrix",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_ndarray",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_random",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_sorting_searching_counting",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_statistics",
"ivy_tests/test_ivy/test_frontends/test_numpy/test_ufunc",
"ivy_tests/test_ivy/test_frontends/test_tensorflow",
"ivy_tests/test_ivy/test_frontends/test_torch",
"ivy_tests/test_ivy/test_functional",
"ivy_tests/test_ivy/test_functional/test_core",
"ivy_tests/test_ivy/test_functional/test_nn",
"ivy_tests/test_ivy/test_stateful",
]
if __name__ == "__main__":
num_tests = len(test_names)
tests_per_run = num_tests // N
start = run_iter * tests_per_run
end = num_tests if run_iter == N - 1 else (run_iter + 1) * tests_per_run
for test_name in tqdm(test_names[start:end]):
os.system(
f"coverage run --source=ivy,ivy_tests -m pytest {test_name} "
"--disable-warnings > coverage_output"
)
os.system("coverage annotate > coverage_output")
for directory in directories:
for file_name in os.listdir(directory):
if file_name.endswith("cover"):
file_name = directory + "/" + file_name
if file_name not in tests:
tests[file_name] = []
with open(file_name) as f:
for line in f:
tests[file_name].append(set())
with open(file_name) as f:
i = 0
for line in f:
if line[0] == ">":
tests[file_name][i].add(
tests["tests_mapping"][test_name]
)
i += 1
os.system("find . -name \\*cover -type f -delete")
commit_hash = ""
for commit in Repository(".", order="reverse").traverse_commits():
commit_hash = commit.hash
break
tests["commit"] = commit_hash
with bz2.BZ2File("tests.pbz2", "w") as f:
cPickle.dump(tests, f)