-
Notifications
You must be signed in to change notification settings - Fork 211
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add JUnit XML support #337
base: master
Are you sure you want to change the base?
Changes from 3 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1930,13 +1930,17 @@ of to a file. | |
@findex srunner_set_xml | ||
@findex srunner_has_xml | ||
@findex srunner_xml_fname | ||
@findex srunner_xml_format | ||
@findex srunner_set_xml_format | ||
The log can also be written in XML. The following functions define | ||
the interface for XML logs: | ||
@example | ||
@verbatim | ||
void srunner_set_xml (SRunner *sr, const char *fname); | ||
int srunner_has_xml (SRunner *sr); | ||
const char *srunner_xml_fname (SRunner *sr); | ||
enum xml_format srunner_xml_format(SRunner * sr); | ||
void srunner_set_xml_format(SRunner * sr, enum xml_format format); | ||
@end verbatim | ||
@end example | ||
|
||
|
@@ -2048,6 +2052,59 @@ If both plain text and XML log files are specified, by any of above methods, | |
then check will log to both files. In other words logging in plain text and XML | ||
format simultaneously is supported. | ||
|
||
JUnit Support is also available. It is enabled by a call to | ||
@code{srunner_set_xml_format(CK_XML_FORMAT_JUNIT)} before the tests are run. | ||
It can also be enabled by environment variable as well. It is enabled by setting the | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could combine two sentences this way:
Also, if a new environment variable is being added, add a reference here: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Addressed in 2e7563c |
||
@code{CK_XML_FORMAT_NAME} environment variable to @code{junit}. | ||
|
||
Here is an example of the JUnit xml format: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. xml -> XML There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Addressed in 2e7563c |
||
@example | ||
@verbatim | ||
<?xml version="1.0" encoding="UTF-8"?> | ||
<testsuites tests="8" errors="1" failures="4"> | ||
<testsuite name="S1" tests="3" errors="1" failures="1"> | ||
<testcase classname="Core" name="test_pass"> | ||
</testcase> | ||
<testcase classname="Core" name="test_fail"> | ||
<failure message="Failure"> | ||
Core:test_fail:0 | ||
ex_output.c:37 | ||
Failure | ||
</failure> | ||
</testcase> | ||
<testcase classname="Core" name="test_exit"> | ||
<error message="Early exit with return value 1"> | ||
Core:test_exit:0 | ||
ex_output.c:46 | ||
Early exit with return value 1 | ||
</error> | ||
</testcase> | ||
</testsuite> | ||
<testsuite name="S2" tests="4" errors="0" failures="2"> | ||
<testcase classname="Core" name="test_pass2"> | ||
</testcase> | ||
<testcase classname="Core" name="test_loop"> | ||
<failure message="Iteration 0 failed"> | ||
Core:test_loop:0 | ||
ex_output.c:72 | ||
Iteration 0 failed | ||
</failure> | ||
</testcase> | ||
<testcase classname="Core" name="test_loop"> | ||
</testcase> | ||
<testcase classname="Core" name="test_loop"> | ||
<failure message="Iteration 2 failed"> | ||
Core:test_loop:2 | ||
ex_output.c:72 | ||
Iteration 2 failed | ||
</failure> | ||
</testcase> | ||
</testsuite> | ||
</testsuites> | ||
@end verbatim | ||
@end example | ||
|
||
|
||
@node TAP Logging, , Test Logging, Test Logging | ||
@subsection TAP Logging | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2272,6 +2272,48 @@ CK_DLL_EXP int CK_EXPORT srunner_has_tap(SRunner * sr); | |
*/ | ||
CK_DLL_EXP const char *CK_EXPORT srunner_tap_fname(SRunner * sr); | ||
|
||
/** | ||
* enum describing the specific XML format used for XML logging | ||
*/ | ||
enum xml_format { | ||
CK_XML_FORMAT_UNSPECIFIED, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should the unspecified state be the default state? What is the motivation for having an unspecified state? And, if it were unspecified what XML stlye ends up being used? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The motivation for the unspecified state is that I needed an initial state which was unspecified since the environment variable to set the format only overrides the selected format if it's unspecified. If it were set to 'default', then I couldn't tell if someone had set it explicitly or not. |
||
CK_XML_FORMAT_DEFAULT, // the default (original) format | ||
CK_XML_FORMAT_JUNIT, // output in JUnit compatible XML | ||
}; | ||
|
||
/** | ||
* Returns the XML format used if XML is to be logged. | ||
* | ||
* This value can be explicitly set via `srunner_set_xml_format` or can | ||
* be set via the CK_XML_FORMAT_NAME environment variable. | ||
* | ||
* @return CK_XML_FORMAT_DEFAULT unless the format is explicitly set via | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If another XML style were introduced in the future, the docs here would need to be updated to remove the JUNIT reference (as there would be other possibilities). Perhaps mention how the environment variable and value set in the code interact instead, so that is more clear. For example, mention that if the environment variable is set then its value is used, otherwise whatever is set in srunner_set_xml_format is used, and the default is CK_XML_FORMAT_DEFAULT. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. addressed in 2e7563c |
||
* `srunner_set_xml_format(sr, CK_XML_FORMAT_JUNIT)` or | ||
* `getenv("CK_XML_FORMAT_NAME")` returns "unit" | ||
* | ||
* @param sr suite runner to check | ||
* | ||
* @since 0.15.3. | ||
*/ | ||
CK_DLL_EXP enum xml_format CK_EXPORT srunner_xml_format(SRunner * sr); | ||
|
||
/** | ||
* Set the suite runner to output the result in an XML format compatible | ||
* with JUnit's XML format. | ||
* | ||
* Note: XML format setting is an initialize only operation -- it should | ||
* be done immediately after SRunner creation, and the XML format can't be | ||
* changed after being set. | ||
* | ||
* This setting does not afffect the other log output types. | ||
* | ||
* @param sr suite runner to log results of in XML format | ||
* @param format the xml_format to use | ||
* | ||
* @since 0.15.3 | ||
*/ | ||
CK_DLL_EXP void CK_EXPORT srunner_set_xml_format(SRunner * sr, enum xml_format format); | ||
|
||
/** | ||
* Enum describing the current fork usage. | ||
*/ | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -59,6 +59,7 @@ struct TCase | |
{ | ||
const char *name; | ||
struct timespec timeout; | ||
struct Suite *s; | ||
List *tflst; /* list of test functions */ | ||
List *unch_sflst; | ||
List *unch_tflst; | ||
|
@@ -82,7 +83,7 @@ struct TestResult | |
int line; /* Line number where the test occurred */ | ||
int iter; /* The iteration value for looping tests */ | ||
int duration; /* duration of this test in microseconds */ | ||
const char *tcname; /* Test case that generated the result */ | ||
TCase *tc; /* Test case that generated the result */ | ||
const char *tname; /* Test that generated the result */ | ||
char *msg; /* Failure message */ | ||
}; | ||
|
@@ -121,6 +122,7 @@ struct SRunner | |
List *resultlst; /* List of unit test results */ | ||
const char *log_fname; /* name of log file */ | ||
const char *xml_fname; /* name of xml output file */ | ||
enum xml_format xml_format; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. there are comments on the other fields, maybe add one for this field too? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Addressed in 2e7563c |
||
const char *tap_fname; /* name of tap output file */ | ||
List *loglst; /* list of Log objects */ | ||
enum fork_status fstat; /* controls if suites are forked or not | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,6 +22,7 @@ | |
|
||
#include <stdlib.h> | ||
#include <stdio.h> | ||
#include <string.h> | ||
#include <check.h> | ||
#if ENABLE_SUBUNIT | ||
#include <subunit/child.h> | ||
|
@@ -63,6 +64,27 @@ const char *srunner_log_fname(SRunner * sr) | |
return getenv("CK_LOG_FILE_NAME"); | ||
} | ||
|
||
enum xml_format srunner_xml_format(SRunner * sr) | ||
{ | ||
// if the format as been explicitly set already via | ||
// `srunner_set_xml_format`, then use that value | ||
if (sr->xml_format != CK_XML_FORMAT_UNSPECIFIED) | ||
return sr->xml_format; | ||
|
||
// junit is the only value of CK_XML_FORMAT_NAME that will | ||
// return something other than CK_XML_FORMAT_DEFAULT | ||
const char *format_name = getenv("CK_XML_FORMAT_NAME"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think in other places in the code the environment variable takes precedence. Example: https://github.com/libcheck/check/blob/master/src/check.c#L317 Should that be the case here as well? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since it's logging, I made the precedence the same as |
||
if (format_name && strcmp(format_name, "junit") == 0) | ||
return CK_XML_FORMAT_JUNIT; | ||
|
||
return CK_XML_FORMAT_DEFAULT; | ||
} | ||
|
||
void srunner_set_xml_format(SRunner * sr, enum xml_format format) | ||
{ | ||
sr->xml_format = format; | ||
} | ||
|
||
|
||
void srunner_set_xml(SRunner * sr, const char *fname) | ||
{ | ||
|
@@ -337,6 +359,65 @@ void xml_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file, | |
|
||
} | ||
|
||
void junit_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file, | ||
enum print_output printmode CK_ATTRIBUTE_UNUSED, void *obj, | ||
enum cl_event evt) | ||
{ | ||
// we're only interested in the end of the full run. | ||
if (evt != CLEND_SR) return; | ||
|
||
TestResult *tr; | ||
Suite *s; | ||
TestStats stats; | ||
|
||
fprintf(file, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); | ||
fprintf(file, | ||
"<testsuites" | ||
" tests=\"%d\"" | ||
" errors=\"%d\"" | ||
" failures=\"%d\"" | ||
">\n", | ||
sr->stats->n_checked, sr->stats->n_errors, sr->stats->n_failed); | ||
|
||
// iterate over the suites | ||
for (check_list_front(sr->slst); !check_list_at_end(sr->slst); check_list_advance(sr->slst)) { | ||
s = (Suite*) check_list_val(sr->slst); | ||
|
||
// calculate the stats | ||
stats.n_checked = stats.n_errors = stats.n_failed = 0; | ||
for (check_list_front(sr->resultlst); !check_list_at_end(sr->resultlst); | ||
check_list_advance(sr->resultlst)) { | ||
tr = (TestResult *)check_list_val(sr->resultlst); | ||
if (tr->tc->s != s) | ||
continue; | ||
stats.n_checked++; | ||
if (tr->rtype == CK_FAILURE) | ||
stats.n_failed++; | ||
else if (tr->rtype == CK_ERROR) | ||
stats.n_errors++; | ||
} | ||
|
||
fprintf(file, " <testsuite name=\""); | ||
fprint_xml_esc(file, s->name); | ||
fprintf(file, | ||
"\"" | ||
" tests=\"%d\"" | ||
" errors=\"%d\"" | ||
" failures=\"%d\"" | ||
">\n", | ||
stats.n_checked, stats.n_errors, stats.n_failed); | ||
for (check_list_front(sr->resultlst); !check_list_at_end(sr->resultlst); | ||
check_list_advance(sr->resultlst)) { | ||
tr = (TestResult *)check_list_val(sr->resultlst); | ||
if (tr->tc->s != s) | ||
continue; | ||
tr_junitprint(file, tr, CK_VERBOSE); | ||
} | ||
fprintf(file, " </testsuite>\n"); | ||
} | ||
fprintf(file, "</testsuites>\n"); | ||
} | ||
|
||
void tap_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file, | ||
enum print_output printmode CK_ATTRIBUTE_UNUSED, void *obj, | ||
enum cl_event evt) | ||
|
@@ -372,7 +453,7 @@ void tap_lfun(SRunner * sr CK_ATTRIBUTE_UNUSED, FILE * file, | |
tr = (TestResult *)obj; | ||
fprintf(file, "%s %d - %s:%s:%s: %s\n", | ||
tr->rtype == CK_PASS ? "ok" : "not ok", num_tests_run, | ||
tr->file, tr->tcname, tr->tname, tr->msg); | ||
tr->file, tr->tc->name, tr->tname, tr->msg); | ||
fflush(file); | ||
break; | ||
default: | ||
|
@@ -520,7 +601,10 @@ void srunner_init_logging(SRunner * sr, enum print_output print_mode) | |
f = srunner_open_xmlfile(sr); | ||
if(f) | ||
{ | ||
srunner_register_lfun(sr, f, f != stdout, xml_lfun, print_mode); | ||
if (srunner_xml_format(sr) == CK_XML_FORMAT_JUNIT) | ||
srunner_register_lfun(sr, f, f != stdout, junit_lfun, print_mode); | ||
else | ||
srunner_register_lfun(sr, f, f != stdout, xml_lfun, print_mode); | ||
} | ||
f = srunner_open_tapfile(sr); | ||
if(f) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is it "JUnit support" or "JUnit XML support" ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Addressed in 2e7563c