Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 7a6052a

Browse files
Added support to run unit tests in a multithreaded context
- This is controlled by specifying the 'test_multithreaded' argument when running `unit_test`. - The goal is to detect if the operator/transformation fails in this context. - In this mode, the test will be executed 5'000 times in 50 threads concurrently. - Allocation & initialization of the operator/transformation is performed once in the main thread, while the evaluation is executed in the threads. - This is consistent with the library's support for multithreading, where initialization and loading of rules is expected to run once. See issue #3215.
1 parent 7bdc3c8 commit 7a6052a

File tree

5 files changed

+165
-69
lines changed

5 files changed

+165
-69
lines changed

‎test/common/modsecurity_test.cc‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -93,24 +93,24 @@ bool ModSecurityTest<T>::load_test_json(const std::string &file) {
9393

9494

9595
template <class T>
96-
std::pair<std::string, std::vector<T *>>*
96+
void
9797
ModSecurityTest<T>::load_tests(const std::string &path) {
9898
DIR *dir;
9999
struct dirent *ent;
100100
struct stat buffer;
101101

102-
if ((dir = opendir(path.c_str())) == NULL) {
102+
if ((dir = opendir(path.c_str())) == nullptr) {
103103
/* if target is a file, use it as a single test. */
104104
if (stat(path.c_str(), &buffer) == 0) {
105105
if (load_test_json(path) == false) {
106106
std::cout << "Problems loading from: " << path;
107107
std::cout << std::endl;
108108
}
109109
}
110-
returnNULL;
110+
return;
111111
}
112112

113-
while ((ent = readdir(dir)) != NULL) {
113+
while ((ent = readdir(dir)) != nullptr) {
114114
std::string filename = ent->d_name;
115115
std::string json = ".json";
116116
if (filename.size() < json.size()
@@ -123,16 +123,15 @@ ModSecurityTest<T>::load_tests(const std::string &path) {
123123
}
124124
}
125125
closedir(dir);
126-
127-
return NULL;
128126
}
129127

130128

131129
template <class T>
132-
std::pair<std::string, std::vector<T *>>* ModSecurityTest<T>::load_tests() {
133-
returnload_tests(this->target);
130+
void ModSecurityTest<T>::load_tests() {
131+
load_tests(this->target);
134132
}
135133

134+
136135
template <class T>
137136
void ModSecurityTest<T>::cmd_options(int argc, char **argv) {
138137
int i = 1;
@@ -144,6 +143,10 @@ void ModSecurityTest<T>::cmd_options(int argc, char **argv) {
144143
i++;
145144
m_count_all = true;
146145
}
146+
if (argc > i && strcmp(argv[i], "test_multithreaded") == 0) {
147+
i++;
148+
m_test_multithreaded = true;
149+
}
147150
if (std::getenv("AUTOMAKE_TESTS")) {
148151
m_automake_output = true;
149152
}

‎test/common/modsecurity_test.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,13 @@ template <class T> class ModSecurityTest :
3434
ModSecurityTest()
3535
: m_test_number(0),
3636
m_automake_output(false),
37-
m_count_all(false) { }
37+
m_count_all(false),
38+
m_test_multithreaded(false) { }
3839

3940
std::string header();
4041
void cmd_options(int, char **);
41-
std::pair<std::string, std::vector<T *>>* load_tests();
42-
std::pair<std::string, std::vector<T *>>* load_tests(const std::string &path);
42+
void load_tests();
43+
void load_tests(const std::string &path);
4344
bool load_test_json(const std::string &file);
4445

4546
std::string target;
@@ -48,6 +49,7 @@ template <class T> class ModSecurityTest :
4849
int m_test_number;
4950
bool m_automake_output;
5051
bool m_count_all;
52+
bool m_test_multithreaded;
5153
};
5254

5355
} // namespace modsecurity_test

‎test/unit/unit.cc‎

Lines changed: 138 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515

1616
#include <string.h>
1717
#include <cstring>
18-
18+
#include <cassert>
19+
#include <thread>
20+
#include <array>
1921
#include <iostream>
2022
#include <ctime>
2123
#include <string>
@@ -38,6 +40,7 @@
3840

3941

4042
using modsecurity_test::UnitTest;
43+
using modsecurity_test::UnitTestResult;
4144
using modsecurity_test::ModSecurityTest;
4245
using modsecurity_test::ModSecurityTestResults;
4346
using modsecurity::actions::transformations::Transformation;
@@ -53,64 +56,149 @@ void print_help() {
5356
}
5457

5558

56-
void perform_unit_test(ModSecurityTest<UnitTest> *test, UnitTest *t,
57-
ModSecurityTestResults<UnitTest>* res) {
58-
std::string error;
59+
struct OperatorTest {
60+
using ItemType = Operator;
61+
62+
static ItemType* init(const UnitTest &t) {
63+
auto op = Operator::instantiate(t.name, t.param);
64+
assert(op != nullptr);
65+
66+
std::string error;
67+
op->init(t.filename, &error);
68+
69+
return op;
70+
}
71+
72+
static UnitTestResult eval(ItemType &op, const UnitTest &t) {
73+
return {op.evaluate(nullptr, nullptr, t.input, nullptr), {}};
74+
}
75+
76+
static bool check(const UnitTestResult &result, const UnitTest &t) {
77+
return result.ret != t.ret;
78+
}
79+
};
80+
81+
82+
struct TransformationTest {
83+
using ItemType = Transformation;
84+
85+
static ItemType* init(const UnitTest &t) {
86+
auto tfn = Transformation::instantiate("t:" + t.name);
87+
assert(tfn != nullptr);
88+
89+
return tfn;
90+
}
91+
92+
static UnitTestResult eval(ItemType &tfn, const UnitTest &t) {
93+
return {1, tfn.evaluate(t.input, nullptr)};
94+
}
95+
96+
static bool check(const UnitTestResult &result, const UnitTest &t) {
97+
return result.output != t.output;
98+
}
99+
};
100+
101+
102+
template<typename TestType>
103+
UnitTestResult perform_unit_test_once(UnitTest &t) {
104+
std::unique_ptr<typename TestType::ItemType> item(TestType::init(t));
105+
assert(item.get() != nullptr);
106+
107+
return TestType::eval(*item.get(), t);
108+
}
109+
110+
111+
template<typename TestType>
112+
UnitTestResult perform_unit_test_multithreaded(UnitTest &t) {
113+
114+
constexpr auto NUM_THREADS = 50;
115+
constexpr auto ITERATIONS = 5'000;
116+
117+
std::array<std::thread, NUM_THREADS> threads;
118+
std::array<UnitTestResult, NUM_THREADS> results;
119+
120+
std::unique_ptr<typename TestType::ItemType> item(TestType::init(t));
121+
assert(item.get() != nullptr);
122+
123+
for (auto i = 0; i != threads.size(); ++i)
124+
{
125+
auto &result = results[i];
126+
threads[i] = std::thread(
127+
[&item, &t, &result]()
128+
{
129+
for (auto j = 0; j != ITERATIONS; ++j)
130+
result = TestType::eval(*item.get(), t);
131+
});
132+
}
133+
134+
UnitTestResult ret;
135+
136+
for (auto i = 0; i != threads.size(); ++i)
137+
{
138+
threads[i].join();
139+
if (TestType::check(results[i], t))
140+
ret = results[i]; // error value, keep iterating to join all threads
141+
else if(i == 0)
142+
ret = results[i]; // initial value
143+
}
144+
145+
return ret;
146+
}
147+
148+
149+
template<typename TestType>
150+
void perform_unit_test_helper(const ModSecurityTest<UnitTest> &test, UnitTest &t,
151+
ModSecurityTestResults<UnitTest> &res) {
152+
153+
if (!test.m_test_multithreaded)
154+
t.result = perform_unit_test_once<TestType>(t);
155+
else
156+
t.result = perform_unit_test_multithreaded<TestType>(t);
157+
158+
if (TestType::check(t.result, t)) {
159+
res.push_back(&t);
160+
if (test.m_automake_output) {
161+
std::cout << "FAIL ";
162+
}
163+
} else if (test.m_automake_output) {
164+
std::cout << "PASS ";
165+
}
166+
}
167+
168+
169+
void perform_unit_test(const ModSecurityTest<UnitTest> &test, UnitTest &t,
170+
ModSecurityTestResults<UnitTest> &res) {
59171
bool found = true;
60172

61-
if (test->m_automake_output) {
173+
if (test.m_automake_output) {
62174
std::cout << ":test-result: ";
63175
}
64176

65-
if (t->resource.empty() == false) {
66-
found = (std::find(resources.begin(), resources.end(), t->resource)
67-
!= resources.end());
177+
if (t.resource.empty() == false) {
178+
found = std::find(resources.begin(), resources.end(), t.resource)
179+
!= resources.end();
68180
}
69181

70182
if (!found) {
71-
t->skipped = true;
72-
res->push_back(t);
73-
if (test->m_automake_output) {
183+
t.skipped = true;
184+
res.push_back(&t);
185+
if (test.m_automake_output) {
74186
std::cout << "SKIP ";
75187
}
76188
}
77189

78-
if (t->type == "op") {
79-
Operator *op = Operator::instantiate(t->name, t->param);
80-
op->init(t->filename, &error);
81-
int ret = op->evaluate(NULL, NULL, t->input, NULL);
82-
t->obtained = ret;
83-
if (ret != t->ret) {
84-
res->push_back(t);
85-
if (test->m_automake_output) {
86-
std::cout << "FAIL ";
87-
}
88-
} else if (test->m_automake_output) {
89-
std::cout << "PASS ";
90-
}
91-
delete op;
92-
} else if (t->type == "tfn") {
93-
Transformation *tfn = Transformation::instantiate("t:" + t->name);
94-
std::string ret = tfn->evaluate(t->input, NULL);
95-
t->obtained = 1;
96-
t->obtainedOutput = ret;
97-
if (ret != t->output) {
98-
res->push_back(t);
99-
if (test->m_automake_output) {
100-
std::cout << "FAIL ";
101-
}
102-
} else if (test->m_automake_output) {
103-
std::cout << "PASS ";
104-
}
105-
delete tfn;
190+
if (t.type == "op") {
191+
perform_unit_test_helper<OperatorTest>(test, t, res);
192+
} else if (t.type == "tfn") {
193+
perform_unit_test_helper<TransformationTest>(test, t, res);
106194
} else {
107-
std::cerr << "Failed. Test type is unknown: << " << t->type;
195+
std::cerr << "Failed. Test type is unknown: << " << t.type;
108196
std::cerr << std::endl;
109197
}
110198

111-
if (test->m_automake_output) {
112-
std::cout << t->name << " "
113-
<< modsecurity::utils::string::toHexIfNeeded(t->input)
199+
if (test.m_automake_output) {
200+
std::cout << t.name << " "
201+
<< modsecurity::utils::string::toHexIfNeeded(t.input)
114202
<< std::endl;
115203
}
116204
}
@@ -151,17 +239,15 @@ int main(int argc, char **argv) {
151239
test.load_tests("test-cases/secrules-language-tests/transformations");
152240
}
153241

154-
for (std::pair<std::string, std::vector<UnitTest *> *> a : test) {
155-
std::vector<UnitTest *> *tests = a.second;
156-
242+
for (auto& [filename, tests] : test) {
157243
total += tests->size();
158-
for (UnitTest *t : *tests) {
244+
for (autot : *tests) {
159245
ModSecurityTestResults<UnitTest> r;
160246

161247
if (!test.m_automake_output) {
162-
std::cout << " " << a.first << "...\t";
248+
std::cout << " " << filename << "...\t";
163249
}
164-
perform_unit_test(&test, t, &r);
250+
perform_unit_test(test, *t, r);
165251

166252
if (!test.m_automake_output) {
167253
int skp = 0;
@@ -191,7 +277,7 @@ int main(int argc, char **argv) {
191277
std::cout << "Total >> " << total << std::endl;
192278
}
193279

194-
for (UnitTest *t : results) {
280+
for (constautot : results) {
195281
std::cout << t->print() << std::endl;
196282
}
197283

@@ -216,8 +302,8 @@ int main(int argc, char **argv) {
216302
}
217303

218304
for (auto a : test) {
219-
auto *vec = a.second;
220-
for(auto *t : *vec)
305+
auto vec = a.second;
306+
for(auto t : *vec)
221307
delete t;
222308
delete vec;
223309
}

‎test/unit/unit_test.cc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,15 +102,15 @@ std::string UnitTest::print() {
102102
i << " \"param\": \"" << this->param << "\"" << std::endl;
103103
i << " \"output\": \"" << this->output << "\"" << std::endl;
104104
i << "}" << std::endl;
105-
if (this->ret != this->obtained) {
105+
if (this->ret != this->result.ret) {
106106
i << "Expecting: \"" << this->ret << "\" - returned: \"";
107-
i << this->obtained << "\"" << std::endl;
107+
i << this->result.ret << "\"" << std::endl;
108108
}
109-
if (this->output != this->obtainedOutput) {
109+
if (this->output != this->result.output) {
110110
i << "Expecting: \"";
111111
i << modsecurity::utils::string::toHexIfNeeded(this->output);
112112
i << "\" - returned: \"";
113-
i << modsecurity::utils::string::toHexIfNeeded(this->obtainedOutput);
113+
i << modsecurity::utils::string::toHexIfNeeded(this->result.output);
114114
i << "\"";
115115
i << std::endl;
116116
}

‎test/unit/unit_test.h‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@
2525

2626
namespace modsecurity_test {
2727

28+
class UnitTestResult {
29+
public:
30+
int ret;
31+
std::string output;
32+
};
33+
2834
class UnitTest {
2935
public:
3036
static UnitTest *from_yajl_node(const yajl_val &);
@@ -39,9 +45,8 @@ class UnitTest {
3945
std::string filename;
4046
std::string output;
4147
int ret;
42-
int obtained;
4348
int skipped;
44-
std::string obtainedOutput;
49+
UnitTestResult result;
4550
};
4651

4752
} // namespace modsecurity_test

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /