Skip to content

Commit

Permalink
Merge pull request #233 from mckeownp/development
Browse files Browse the repository at this point in the history 
Updates to utility scripts to play well with updated bulk_tester class
  • Loading branch information
@trampgeek
trampgeek authored Jan 21, 2025
2 parents f0a1a2f + de99be4 commit 157bca2
Show file tree
Hide file tree
Showing 10 changed files with 76 additions and 61 deletions.
2 changes: 1 addition & 1 deletion bulktest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
// Get the parameters from the URL.
$contextid = required_param('contextid', PARAM_INT); // Set to 0 if providing a list of question IDs to check.
$categoryid = optional_param('categoryid', null, PARAM_INT);
$randomseed = optional_param('randomseed', 0, PARAM_INT);
$randomseed = optional_param('randomseed', -1, PARAM_INT);
$repeatrandomonly = optional_param('repeatrandomonly', 1, PARAM_INT);
$nruns = optional_param('nruns', 1, PARAM_INT);
$questionids = optional_param('questionids', '', PARAM_RAW); // A list of specific questions to check, eg, for rechecking failed tests.
Expand Down
13 changes: 7 additions & 6 deletions bulktestall.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@
$title = get_string('bulktesttitle', 'qtype_coderunner', $context->get_context_name());
$PAGE->set_title($title);

// Create the helper class.
$bulktester = new qtype_coderunner_bulk_tester();

$numpasses = 0;
$allfailingtests = [];
$allmissinganswers = [];
Expand All @@ -60,14 +59,16 @@
echo $OUTPUT->heading($title, 1);

// Run the tests.
foreach ($bulktester->get_num_coderunner_questions_by_context() as $contextid => $numcoderunnerquestions) {
$contextdata = qtype_coderunner_bulk_tester::get_num_coderunner_questions_by_context();
foreach ($contextdata as $contextid => $numcoderunnerquestions) {
if ($skipping && $contextid != $startfromcontextid) {
continue;
}
$skipping = false;

$testcontext = context::instance_by_id($contextid);
if (has_capability('moodle/question:editall', $context)) {
$PAGE->set_context($testcontext); // Helps grading cache pickup right course id.
$bulktester = new qtype_coderunner_bulk_tester($testcontext);
echo $OUTPUT->heading(get_string('bulktesttitle', 'qtype_coderunner', $testcontext->get_context_name()));
echo html_writer::tag('p', html_writer::link(
new moodle_url(
Expand All @@ -77,13 +78,13 @@
get_string('bulktestcontinuefromhere', 'qtype_coderunner')
));

[$passes, $failingtests, $missinganswers] = $bulktester->run_all_tests_for_context($testcontext);
[$passes, $failingtests, $missinganswers] = $bulktester->run_all_tests_for_context();
$numpasses += $passes;
$allfailingtests = array_merge($allfailingtests, $failingtests);
$allmissinganswers = array_merge($allmissinganswers, $missinganswers);
}
}

// Display the final summary.
$bulktester->print_overall_result($numpasses, $allfailingtests, $allmissinganswers);
qtype_coderunner_bulk_tester::print_summary_after_bulktestall($numpasses, $allfailingtests, $allmissinganswers);
echo $OUTPUT->footer();
33 changes: 16 additions & 17 deletions bulktestindex.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,10 @@
<div style="margin-bottom: 10px; display: grid; grid-template-columns: auto 80px; gap: 10px; align-items: center; max-width: 240px;">
<label for="nruns">Number of runs:</label>
<input type="number" id="nruns" value="{$nruns}" min="1" style="width: 80px;">
<label for="randomseed">Random seed:</label>
<input type="number" id="randomseed" value="0" min="0" style="width: 80px;">
<input type="number" id="randomseed" value="" min="0" style="width: 80px;">
<label for="repeatrandomonly">Repeat random only:</label>
<div>
<input type="checkbox" id="repeatrandomonly" checked>
Expand All @@ -86,25 +86,23 @@

$testallstr = get_string('bulktestallincontext', 'qtype_coderunner');
$testalltitledetails = ['title' => get_string('testalltitle', 'qtype_coderunner'), 'style' => $buttonstyle];
$testallspan = html_writer::tag('span', $testallstr,
['class' => 'test-link',
$testallspan = html_writer::tag(
'span', $testallstr,
['class' => 'test-link',
'data-contextid' => $contextid,
'style' => $buttonstyle . ';cursor:pointer;']);

'style' => $buttonstyle . ';cursor:pointer;']
);
$expandlink = html_writer::link(
'#expand',
get_string('expand', 'qtype_coderunner'),
['class' => 'expander', 'title' => get_string('expandtitle', 'qtype_coderunner'), 'style' => $buttonstyle]
);

$litext = $name . ' (' . $numcoderunnerquestions . ') ' . $testallspan . ' ' . $expandlink;

if (strpos($name, ": Quiz: ") === false) {
$class = 'bulktest coderunner context normal';
} else {
$class = 'bulktest coderunner context quiz';
}

echo html_writer::start_tag('li', ['class' => $class]);
echo $litext;

Expand All @@ -115,11 +113,12 @@
foreach ($categories as $cat) {
if ($cat->count > 0) {
$linktext = $cat->name . ' (' . $cat->count . ')';
$span = html_writer::tag('span', $linktext,
$span = html_writer::tag('span', $linktext,
['class' => 'test-link',
'data-contextid' => $contextid,
'data-categoryid' => $cat->id,
'style' => $buttonstyle . ';cursor:pointer;']);
'style' => $buttonstyle . ';cursor:pointer;']
);
echo html_writer::tag('li', $span, $titledetails);
}
}
Expand All @@ -145,7 +144,7 @@
Array.from(expandables).forEach(function (expandable) {
expandable.style.display = 'none';
});
var expanders = document.getElementsByClassName('expander');
Array.from(expanders).forEach(function(expander) {
expander.addEventListener('click', function(event) {
Expand All @@ -165,24 +164,24 @@
Array.from(testLinks).forEach(function(link) {
link.addEventListener('click', function(event) {
event.preventDefault();
// Get configuration values
var nruns = document.getElementById('nruns').value;
var randomseed = document.getElementById('randomseed').value;
var repeatrandomonly = document.getElementById('repeatrandomonly').checked ? 1 : 0;
// Build URL parameters
var params = new URLSearchParams();
params.append('contextid', link.dataset.contextid);
params.append('randomseed', randomseed);
params.append('repeatrandomonly', repeatrandomonly);
params.append('nruns', nruns);
// Add category ID if present
if (link.dataset.categoryid) {
params.append('categoryid', link.dataset.categoryid);
}
// Construct and navigate to URL
var url = M.cfg.wwwroot + '/question/type/coderunner/bulktest.php?' + params.toString();
window.location.href = url;
Expand Down
51 changes: 40 additions & 11 deletions classes/bulk_tester.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ class qtype_coderunner_bulk_tester {
/**
* @param context $context the context to run the tests for.
* @param int $categoryid test only questions in this category. Default to all.
* @param int $randomseed used to set random seed before runs for each question. Default = 0 --- meaning seed is not set.
* @param int $randomseed used to set random seed before runs for each question. Default = -1 --- meaning seed is not set.
* Use this to have more chance of the series of questions being generated for testing is the same for a new run
* of the tests. This works well with grader caching as you won't keep getting new random variations.
* Also allows you to mix up the space that is being tested.
Expand All @@ -80,12 +80,16 @@ class qtype_coderunner_bulk_tester {
* @param int $nruns the number times to test each question. Default to 1.
*/
public function __construct(
$context,
$context = null,
$categoryid = null,
$randomseed = 0,
$randomseed = -1,
$repeatrandomonly = 1,
$nruns = 1
) {
if ($context === null) {
$site = get_site(); // Get front page course.
$context = context_course::instance($site->id);
}
$this->context = $context;
$this->categoryid = $categoryid;
$this->randomseed = $randomseed;
Expand Down Expand Up @@ -309,7 +313,7 @@ public function run_all_tests_for_context($questionidstoinclude = []) {
} else {
$qparams['courseid'] = SITEID;
}
$questiontestsurl = new moodle_url('/question/type/coderunner/questiontestrun.php');
$questiontestsurl = new moodle_url('/question/type/coderunner/questiontestrun.php');
$questiontestsurl->params($qparams);

$this->numpasses = 0;
Expand Down Expand Up @@ -345,12 +349,9 @@ public function run_all_tests_for_context($questionidstoinclude = []) {
} else {
$nrunsthistime = $this->nruns;
}
if ($this->randomseed > 0) {
if ($this->randomseed >= 0) {
mt_srand($this->randomseed);
}
if ($question->id == 6273) {
echo $question->id;
}
// Now run the test for the required number of times.
for ($i = 0; $i < $nrunsthistime; $i++) {
// Only records last outcome and message.
Expand Down Expand Up @@ -403,7 +404,7 @@ public function run_all_tests_for_context($questionidstoinclude = []) {
}
echo "</ul>\n";
}
return;
return [$this->numpasses, $this->failedtestdetails, $this->missinganswerdetails];
}


Expand Down Expand Up @@ -496,8 +497,6 @@ public function print_overall_result() {
}
echo html_writer::end_tag('ul');
}


if (count($this->failedtestdetails) > 0) {
echo $OUTPUT->heading(get_string('coderunner_install_testsuite_failures', 'qtype_coderunner'), 5);
echo html_writer::start_tag('ul');
Expand Down Expand Up @@ -531,6 +530,36 @@ public function print_overall_result() {
echo html_writer::tag('p', $link);
}

/**
* Print an overall summary of the failed tests.
*/
public static function print_summary_after_bulktestall($numpasses, $allfailingtests, $allmissinganswers) {
global $OUTPUT;
echo $OUTPUT->heading(get_string('bulktestoverallresults', 'qtype_coderunner'), 5);
$spacer = '&nbsp;&nbsp;|&nbsp;&nbsp;';
$passstr = $numpasses . ' ' . get_string('passes', 'qtype_coderunner') . $spacer;
$failstr = count($allfailingtests) . ' ' . get_string('fails', 'qtype_coderunner') . $spacer;
$missingstr = count($allmissinganswers) . ' ' . get_string('missinganswers', 'qtype_coderunner');
echo html_writer::tag('p', $passstr . $failstr . $missingstr);

if (count($allmissinganswers) > 0) {
echo $OUTPUT->heading(get_string('coderunner_install_testsuite_noanswer', 'qtype_coderunner'), 5);
echo html_writer::start_tag('ul');
foreach ($allmissinganswers as $message) {
echo html_writer::tag('li', $message);
}
echo html_writer::end_tag('ul');
}
if (count($allfailingtests) > 0) {
echo $OUTPUT->heading(get_string('coderunner_install_testsuite_failures', 'qtype_coderunner'), 5);
echo html_writer::start_tag('ul');
foreach ($allfailingtests as $message) {
echo html_writer::tag('li', $message);
}
echo html_writer::end_tag('ul');
}
}


/**
* Display the results of scanning all the CodeRunner questions to
Expand Down
2 changes: 1 addition & 1 deletion classes/jobesandbox.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ public function execute($sourcecode, $language, $input, $files = null, $params =
try {
// Had to use try here as isset($PAGE->context) always seems to fail even if the context has been set.
$context = $PAGE->context;
$courseid = $context->get_course_context(strict: true)->instanceid;
$courseid = $context->get_course_context(true)->instanceid; // raises exception if context is unknown.
} catch (Exception $e) {
$courseid = 1; // Use context of 1 as no $PAGE context is set, eg, could be a websocket UI run.
}
Expand Down
4 changes: 1 addition & 3 deletions downloadquizattempts.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,7 @@
$PAGE->requires->jquery_plugin('ui');
$PAGE->requires->jquery_plugin('ui-css');

// Create the helper class.
$bulktester = new qtype_coderunner_bulk_tester();
$courses = $bulktester->get_all_courses();
$courses = qtype_coderunner_bulk_tester::get_all_courses();

// Start display.
echo $OUTPUT->header();
Expand Down
8 changes: 1 addition & 7 deletions findduplicates.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,20 +50,14 @@
$PAGE->set_cm($cm, $DB->get_record('course', ['id' => $cm->course], '*', MUST_EXIST));
}

// Create the helper class.
$bulktester = new qtype_coderunner_bulk_tester();

// Release the session, so the user can do other things while this runs.
\core\session\manager::write_close();

// Display.
echo $OUTPUT->header();
echo $OUTPUT->heading($title);

echo "<table class='table table-bordered table-striped'>\n";
echo "<tr><th>Q1 name</th><th>Q1 Category</th><th>Q2 name</th><th>Q2 category</th></tr>\n";
// Find all the duplicates.
$allquestionsmap = $bulktester->get_all_coderunner_questions_in_context($contextid);
$allquestionsmap = qtype_coderunner_bulk_tester::get_all_coderunner_questions_in_context($contextid);
$allquestions = array_values($allquestionsmap);
$numduplicates = 0;
for ($i = 0; $i < count($allquestions); $i++) {
Expand Down
5 changes: 1 addition & 4 deletions findduplicatesindex.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,12 @@
$PAGE->set_context($context);
$PAGE->set_title('Find duplicate questions');

// Create the helper class.
$bulktester = new qtype_coderunner_bulk_tester();

// Display.
echo $OUTPUT->header();
echo $OUTPUT->heading('Courses containing CodeRunner questions');

// Find in which contexts the user can edit questions.
$questionsbycontext = $bulktester->get_num_coderunner_questions_by_context();
$questionsbycontext = qtype_coderunner_bulk_tester::get_num_coderunner_questions_by_context();
$availablequestionsbycontext = [];
foreach ($questionsbycontext as $contextid => $numcoderunnerquestions) {
$context = context::instance_by_id($contextid);
Expand Down
15 changes: 7 additions & 8 deletions lang/en/qtype_coderunner.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1374,18 +1374,17 @@ function should be applied, e.g. <code>{{STUDENT_ANSWER | e(\'py\')}}</code> is

$string['purgeoldcachekeysbutton'] = 'Purge only OLD keys';
$string['purgeallcachekeysbutton'] = 'Purge ALL keys';
$string['bulktestinfo'] = 'By default clicking on test links will test with <b>...&randomseed=0&repeatrandomonly=1&nruns=1</b>. Feel free to change the url as you see fit.<br>
By default each question with \'random\' in the name will be tested <i>nruns</i> times
unless <i>repeatrandomonly</i> is 0, in which case all questions are tested that many times.<br>A randomseed of 0 means the seed is not set.
Setting it to another value (eg, 1) will mean that the php randomseed is set before doing the test(s) for each question with random in its name.<br>
If you don\'t set the random seed then each time you do a bulk test you will get random sequences of question instances, which may mean the grade
cache isn\'t so useful and more questions have to actually be run on the Jobe server (depending on how random your questions are).<br>
$string['bulktestinfo'] = '<ul>
<li><b>Number of runs:</b> How many times each included question will be tested.</li>
<li><b>Repeat random only:</b> Limits repeated runs to questions with <emph>random</emph> in their names.</li>
<li><b>randomseed:</b> If set to a postive integer then the PHP randomseed is set to this value before running the test(s) for each question that has <emph>random</emph> in its name.<br>
If you don\'t set the random seed then each time you do a bulk test you will get random sequences of question instances, which may mean the grade cache isn\'t so useful and more questions have to actually be run on the Jobe server (depending on how random your questions are).<br>
For a given seed the sequence of random question instances should be the same (assuming your question template uses the random seed it is given correctly).
Setting the random seed allows you to recreate a specific sequences of random question instances (eg, you could do 100 runs with 1 being
the initial seed, then try 100 runs with 200 being the seed, etc, hopefully getting more coverage).';
the initial seed, then try 100 runs with 200 being the seed, etc, hopefully getting more coverage).</li>';
$string['cachepurgeindextitle'] = 'Coderunner Cache Purge Index';
$string['cachepurgepagetitle'] = 'Purging cache for {$a}';
$string['cachepurgeindexinfo'] = 'Purging OLD keys will only delete cache entries for grading runs that are older than the Coderunner cache Time To Live (TTL) as set in the admin settings. Pruging ALL will delete all cache entries for the given course.';
$string['cachepurgeindexinfo'] = 'Purging OLD keys will only delete cache entries for grading runs that are older than the Coderunner cache Time To Live (TTL) as set in the admin settings. Purging ALL will delete all cache entries for the given course.';
$string['currentttlinfo'] = 'Coderunner grading cache Time to Live is currently set to TTL = {$a->seconds} seconds (={$a->days} days)';
$string['unauthorisedcachepurging'] = 'You do not have suitable access to any CodeRunner questions!';
$string['contextidnotacourse'] = 'Nothing to do as context_id $contextid is not a course.';
4 changes: 1 addition & 3 deletions prototypeusageindex.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,7 @@
$PAGE->set_context($context);
$PAGE->set_title(get_string('prototypeusageindex', 'qtype_coderunner'));

// Create the helper class.
$bulktester = new qtype_coderunner_bulk_tester();
$allcourses = $bulktester->get_all_courses();
$allcourses = qtype_coderunner_bulk_tester::get_all_courses();

// Start display.
echo $OUTPUT->header();
Expand Down

0 comments on commit 157bca2

Please sign in to comment.