From b7d01a50b259b71ab7b6e6e24d053f4e1377207a Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Tue, 2 May 2017 16:21:13 -0700 Subject: [PATCH 01/26] add slipcalc form --- server/constants.py | 2 ++ server/forms.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/server/constants.py b/server/constants.py index 3e5443757..3604eb439 100644 --- a/server/constants.py +++ b/server/constants.py @@ -20,6 +20,8 @@ 'regrade', 'revision', 'checkpoint 1', 'checkpoint 2', 'private', 'autograder', 'error'] +TIMESCALES = ['days', 'hours', 'minutes'] + API_PREFIX = '/api' OAUTH_SCOPES = ['all', 'email'] OAUTH_OUT_OF_BAND_URI = 'urn:ietf:wg:oauth:2.0:oob' diff --git a/server/forms.py b/server/forms.py index f5a872174..db29af4bd 100644 --- a/server/forms.py +++ b/server/forms.py @@ -15,7 +15,7 @@ from server import utils import server.canvas.api as canvas_api from server.models import Assignment, User, Client, Course, Message, CanvasCourse -from server.constants import (SCORE_KINDS, COURSE_ENDPOINT_FORMAT, +from server.constants import (SCORE_KINDS, TIMESCALES, COURSE_ENDPOINT_FORMAT, TIMEZONE, STUDENT_ROLE, ASSIGNMENT_ENDPOINT_FORMAT, COMMON_LANGUAGES, ROLE_DISPLAY_NAMES, OAUTH_OUT_OF_BAND_URI) @@ -558,6 +558,17 @@ class ExportAssignment(BaseForm): anonymize = BooleanField('Anonymize', default=False, description="Enable to remove identifying information from submissions") + +class SlipCalculatorForm(BaseForm): + timescale = SelectField('Time Scale', default="Days", + choices=[(c, c.title()) for c in TIMESCALES], + validators=[validators.required()], + description=["Select time scale for slip calculation (slip days, hours, minutes)"]) + grace_amount = DecimalField('Grace Amount', default=0) + grace_scale = SelectField('Grace Scale', default="Minutes", + choices=[(c, c.title()) for c in TIMESCALES], + description=["Select time scale for grace amount (grace days, hours, minutes"]) + ########## # Canvas # ########## From 9f0fe7582e14b60ce3832e19cdc23a477a90afa0 Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Tue, 2 May 2017 16:40:21 -0700 Subject: [PATCH 02/26] add link to calculate slips --- server/controllers/admin.py | 39 +++++++++++++++++ server/forms.py | 6 +-- .../staff/course/assignment/assignment.html | 3 ++ .../templates/staff/jobs/slip_calculator.html | 42 +++++++++++++++++++ 4 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 server/templates/staff/jobs/slip_calculator.html diff --git a/server/controllers/admin.py b/server/controllers/admin.py index 8b091a970..6f9fdde61 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -974,6 +974,45 @@ def checkpoint_grading(cid, aid): form=form, ) +@admin.route("/course//assignments//slip", + methods=["GET", "POST"]) +@is_staff(course_arg='cid') +def slip_calculator(cid, aid): + courses, current_course = get_courses(cid) + assign = Assignment.query.filter_by(id=aid, course_id=cid).one_or_none() + # if not assign or not Assignment.can(assign, current_user, 'grade'): + # flash('Cannot access assignment', 'error') + # return abort(404) + + form = forms.SlipCalculatorForm() + # if form.validate_on_submit(): + # job = jobs.enqueue_job( + # checkpoint.assign_scores, + # description='Checkpoint Scoring for {}'.format(assign.display_name), + # timeout=600, + # course_id=cid, + # user_id=current_user.id, + # assign_id=assign.id, + # score=form.score.data, + # kind=form.kind.data, + # message=form.message.data, + # deadline=form.deadline.data, + # include_backups=form.include_backups.data) + # return redirect(url_for('.course_job', cid=cid, job_id=job.id)) + # else: + # if not form.kind.data: + # form.kind.default = 'checkpoint 1' + # if not form.deadline.data: + # form.deadline.default = utils.local_time_obj(assign.due_date, assign.course) + # form.process() + + return render_template( + 'staff/jobs/slip_calculator.html', + courses=courses, + current_course=current_course, + assignment=assign, + form=form, + ) ############## # Enrollment # diff --git a/server/forms.py b/server/forms.py index db29af4bd..603c2c9f4 100644 --- a/server/forms.py +++ b/server/forms.py @@ -563,11 +563,7 @@ class SlipCalculatorForm(BaseForm): timescale = SelectField('Time Scale', default="Days", choices=[(c, c.title()) for c in TIMESCALES], validators=[validators.required()], - description=["Select time scale for slip calculation (slip days, hours, minutes)"]) - grace_amount = DecimalField('Grace Amount', default=0) - grace_scale = SelectField('Grace Scale', default="Minutes", - choices=[(c, c.title()) for c in TIMESCALES], - description=["Select time scale for grace amount (grace days, hours, minutes"]) + description="Time scale for slip calculation.") ########## # Canvas # diff --git a/server/templates/staff/course/assignment/assignment.html b/server/templates/staff/course/assignment/assignment.html index 2904f57f8..a56852898 100644 --- a/server/templates/staff/course/assignment/assignment.html +++ b/server/templates/staff/course/assignment/assignment.html @@ -92,6 +92,9 @@

Actions

  • Grant Extension
    • +
  • + Calculate Slips +
  • Configure Autograder
    • diff --git a/server/templates/staff/jobs/slip_calculator.html b/server/templates/staff/jobs/slip_calculator.html new file mode 100644 index 000000000..912f8db5b --- /dev/null +++ b/server/templates/staff/jobs/slip_calculator.html @@ -0,0 +1,42 @@ +{% extends "staff/base.html" %} +{% import "staff/_formhelpers.html" as forms %} + +{% block title %} Calculate Slips for {{ assignment.display_name }} {% endblock %} + +{% block main %} +
    +

    + Calculate Slips for {{ assignment.display_name }} + {{ current_course.offering }} +

    +
      +
    1. + {{ current_course.offering }} +
      2. +
    2. + Assignments +
      3. +
    3. {{ assignment.display_name }}
      4. +
    4. + Jobs +
      5. +
    5. + Calculate Slips +
      6. +
    +
    +
    + {% include 'alerts.html' %} +
    +
    +
    +
    + {% call forms.render_form(form, action_text='Calculate Slips') %} + {{ forms.render_field(form.timescale) }} + {% endcall %} +
    +
    +
    +
    +
    +{% endblock %} From d6cbe8caece0066ebe1bd742266f36193439031c Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Tue, 2 May 2017 17:07:09 -0700 Subject: [PATCH 03/26] setup basic job scaffold --- server/controllers/admin.py | 48 +++++++++---------- server/jobs/slips.py | 8 ++++ .../staff/course/assignment/assignment.html | 2 +- ...p_calculator.html => calculate_slips.html} | 0 4 files changed, 31 insertions(+), 27 deletions(-) create mode 100644 server/jobs/slips.py rename server/templates/staff/jobs/{slip_calculator.html => calculate_slips.html} (100%) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index 6f9fdde61..c42902da9 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -30,7 +30,7 @@ import server.forms as forms import server.jobs as jobs from server.jobs import (example, export, moss, scores_audit, github_search, - scores_notify, checkpoint) + scores_notify, checkpoint, slips) import server.highlight as highlight import server.utils as utils @@ -977,37 +977,33 @@ def checkpoint_grading(cid, aid): @admin.route("/course//assignments//slip", methods=["GET", "POST"]) @is_staff(course_arg='cid') -def slip_calculator(cid, aid): +def calculate_slips(cid, aid): courses, current_course = get_courses(cid) assign = Assignment.query.filter_by(id=aid, course_id=cid).one_or_none() - # if not assign or not Assignment.can(assign, current_user, 'grade'): - # flash('Cannot access assignment', 'error') - # return abort(404) + if not assign or not Assignment.can(assign, current_user, 'grade'): + flash('Cannot access assignment', 'error') + return abort(404) form = forms.SlipCalculatorForm() - # if form.validate_on_submit(): - # job = jobs.enqueue_job( - # checkpoint.assign_scores, - # description='Checkpoint Scoring for {}'.format(assign.display_name), - # timeout=600, - # course_id=cid, - # user_id=current_user.id, - # assign_id=assign.id, - # score=form.score.data, - # kind=form.kind.data, - # message=form.message.data, - # deadline=form.deadline.data, - # include_backups=form.include_backups.data) - # return redirect(url_for('.course_job', cid=cid, job_id=job.id)) - # else: - # if not form.kind.data: - # form.kind.default = 'checkpoint 1' - # if not form.deadline.data: - # form.deadline.default = utils.local_time_obj(assign.due_date, assign.course) - # form.process() + if form.validate_on_submit(): + print("submitted!") + job = jobs.enqueue_job( + slips.calculate_slips, + description='Calculate Slips for {}'.format(assign.display_name), + # timeout=600, + course_id=cid, + # user_id=current_user.id, + # assign_id=assign.id, + # score=form.score.data, + # kind=form.kind.data, + # message=form.message.data, + # deadline=form.deadline.data, + # include_backups=form.include_backups.data + ) + return redirect(url_for('.course_job', cid=cid, job_id=job.id)) return render_template( - 'staff/jobs/slip_calculator.html', + 'staff/jobs/calculate_slips.html', courses=courses, current_course=current_course, assignment=assign, diff --git a/server/jobs/slips.py b/server/jobs/slips.py new file mode 100644 index 000000000..56666b3df --- /dev/null +++ b/server/jobs/slips.py @@ -0,0 +1,8 @@ + + +from server import jobs + +@jobs.background_job +def calculate_slips(): + logger = jobs.get_job_logger() + logger.info('hello world!') diff --git a/server/templates/staff/course/assignment/assignment.html b/server/templates/staff/course/assignment/assignment.html index a56852898..3d192fe61 100644 --- a/server/templates/staff/course/assignment/assignment.html +++ b/server/templates/staff/course/assignment/assignment.html @@ -92,7 +92,7 @@

    Actions

  • Grant Extension
    • -
  • +
  • Calculate Slips
  • diff --git a/server/templates/staff/jobs/slip_calculator.html b/server/templates/staff/jobs/calculate_slips.html similarity index 100% rename from server/templates/staff/jobs/slip_calculator.html rename to server/templates/staff/jobs/calculate_slips.html From 5d5e6a9b1d003e7210e46c5b631dc24bc584deab Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Wed, 3 May 2017 17:01:43 -0700 Subject: [PATCH 04/26] logic for slip calculation --- server/controllers/admin.py | 8 +++++--- server/jobs/slips.py | 25 +++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index c42902da9..fee5399cc 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -985,15 +985,17 @@ def calculate_slips(cid, aid): return abort(404) form = forms.SlipCalculatorForm() + timescale = form.timescale.data.title() if form.validate_on_submit(): print("submitted!") job = jobs.enqueue_job( slips.calculate_slips, - description='Calculate Slips for {}'.format(assign.display_name), - # timeout=600, + description='Calculate Slip {} for {}'.format(timescale, assign.display_name), + timeout=600, course_id=cid, # user_id=current_user.id, - # assign_id=assign.id, + assign_id=assign.id, + timescale=timescale, # score=form.score.data, # kind=form.kind.data, # message=form.message.data, diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 56666b3df..8f127b93d 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -1,8 +1,29 @@ from server import jobs +from server.models import Assignment +from server.constants import TIMESCALES +from datetime import datetime as dt +from math import ceil + +timescales = {'days':86400, 'hours':3600, 'minutes':60} + +def timediff(created, deadline, timescale): + secs_over = (created - deadline).total_seconds() + return ceil(secs_over / timescales[timescale.lower()]) @jobs.background_job -def calculate_slips(): +def calculate_slips(assign_id, timescale): logger = jobs.get_job_logger() - logger.info('hello world!') + job = jobs.get_current_job() + + logger.info('Calculating slip {}...'.format(timescale)) + assignment = Assignment.query.get(assign_id) + subms = assignment.course_submissions(include_empty=False) + deadline = assignment.due_date + for subm in subms: + email = subm['user']['email'] + created = subm['backup']['created'] + logger.info('{} {}' + .format(email, timediff(created, deadline, timescale))) + From 82c50fa880b2a263d10f00fe436fff087dacf68a Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Wed, 3 May 2017 23:11:16 -0700 Subject: [PATCH 05/26] output csv string --- server/controllers/admin.py | 7 +------ server/jobs/slips.py | 39 ++++++++++++++++++++++++++++--------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index fee5399cc..108b1bebf 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -993,14 +993,9 @@ def calculate_slips(cid, aid): description='Calculate Slip {} for {}'.format(timescale, assign.display_name), timeout=600, course_id=cid, - # user_id=current_user.id, + user_id=current_user.id, assign_id=assign.id, timescale=timescale, - # score=form.score.data, - # kind=form.kind.data, - # message=form.message.data, - # deadline=form.deadline.data, - # include_backups=form.include_backups.data ) return redirect(url_for('.course_job', cid=cid, job_id=job.id)) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 8f127b93d..a8db7131d 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -1,29 +1,50 @@ - +from datetime import datetime as dt +import math +import io +import csv from server import jobs from server.models import Assignment from server.constants import TIMESCALES -from datetime import datetime as dt -from math import ceil timescales = {'days':86400, 'hours':3600, 'minutes':60} def timediff(created, deadline, timescale): secs_over = (created - deadline).total_seconds() - return ceil(secs_over / timescales[timescale.lower()]) + return math.ceil(secs_over / timescales[timescale.lower()]) + +def csv_data(header, rows): + output = io.StringIO() + writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC) + writer.writerow(header) + [writer.writerow(row) for row in rows] + return output.getvalue() @jobs.background_job def calculate_slips(assign_id, timescale): logger = jobs.get_job_logger() job = jobs.get_current_job() - - logger.info('Calculating slip {}...'.format(timescale)) + + logger.info('Calculating Slip {}...'.format(timescale.title())) + assignment = Assignment.query.get(assign_id) subms = assignment.course_submissions(include_empty=False) deadline = assignment.due_date - for subm in subms: + + def get_row(subm): email = subm['user']['email'] created = subm['backup']['created'] - logger.info('{} {}' - .format(email, timediff(created, deadline, timescale))) + slips = min(0, timediff(created, deadline, timescale)) + return [email, slips] + + header = ('User Email', 'Slip {} Used'.format(timescale.title())) + rows = (get_row(subm) for subm in subms) + data = csv_data(header, rows) + + logger.info(repr(data)) + + # upload = ExternalFile.upload(csv_data, user_id=1, course_id=1, + # name='temp.okfile', prefix='jobs/example/') + + From a10376ae52075a3ddb88b6147bb0c73beebe84f8 Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Thu, 4 May 2017 00:55:23 -0700 Subject: [PATCH 06/26] save csf file - complete assign route --- server/controllers/admin.py | 2 + server/forms.py | 1 + server/jobs/slips.py | 40 +++++++++++-------- .../templates/staff/jobs/calculate_slips.html | 2 + server/utils.py | 10 +++++ 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index 108b1bebf..434adf0a1 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -996,6 +996,8 @@ def calculate_slips(cid, aid): user_id=current_user.id, assign_id=assign.id, timescale=timescale, + show_results=form.show_results.data, + result_kind='link', ) return redirect(url_for('.course_job', cid=cid, job_id=job.id)) diff --git a/server/forms.py b/server/forms.py index 603c2c9f4..924fae3dd 100644 --- a/server/forms.py +++ b/server/forms.py @@ -564,6 +564,7 @@ class SlipCalculatorForm(BaseForm): choices=[(c, c.title()) for c in TIMESCALES], validators=[validators.required()], description="Time scale for slip calculation.") + show_results = BooleanField('Show Results', default=False) ########## # Canvas # diff --git a/server/jobs/slips.py b/server/jobs/slips.py index a8db7131d..c28ff892f 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -4,7 +4,8 @@ import csv from server import jobs -from server.models import Assignment +from server.models import Assignment, ExternalFile +from server.utils import encode_id, local_time, output_csv_iterable from server.constants import TIMESCALES timescales = {'days':86400, 'hours':3600, 'minutes':60} @@ -13,21 +14,15 @@ def timediff(created, deadline, timescale): secs_over = (created - deadline).total_seconds() return math.ceil(secs_over / timescales[timescale.lower()]) -def csv_data(header, rows): - output = io.StringIO() - writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC) - writer.writerow(header) - [writer.writerow(row) for row in rows] - return output.getvalue() - @jobs.background_job -def calculate_slips(assign_id, timescale): +def calculate_slips(assign_id, timescale, show_results): logger = jobs.get_job_logger() job = jobs.get_current_job() logger.info('Calculating Slip {}...'.format(timescale.title())) assignment = Assignment.query.get(assign_id) + course = assignment.course subms = assignment.course_submissions(include_empty=False) deadline = assignment.due_date @@ -39,12 +34,25 @@ def get_row(subm): header = ('User Email', 'Slip {} Used'.format(timescale.title())) rows = (get_row(subm) for subm in subms) - data = csv_data(header, rows) - - logger.info(repr(data)) - - # upload = ExternalFile.upload(csv_data, user_id=1, course_id=1, - # name='temp.okfile', prefix='jobs/example/') - + logger.info('Outputting csv...\n') + csv_iterable = output_csv_iterable(header, rows) + + logger.info('Uploading...') + created_time = local_time(dt.now(), course, fmt='%m-%d-%I-%M-%p') + csv_name = '{}_{}.csv'.format(assignment.name.replace('/', '-'), created_time) + upload = ExternalFile.upload(csv_iterable, + user_id=job.user.id, course_id=course.id, name=csv_name, + prefix='jobs/slips/{}'.format(course.offering)) + logger.info('Saved as: {}'.format(upload.object_name)) + + download_link = "/files/{}".format(encode_id(upload.id)) + logger.info('Download link: {} (see "result" above)\n'.format(download_link)) + + if show_results: + logger.info('Results:\n') + csv_data = ''.join([row.decode('utf-8') for row in csv_iterable]) + logger.info(csv_data) + + return download_link diff --git a/server/templates/staff/jobs/calculate_slips.html b/server/templates/staff/jobs/calculate_slips.html index 912f8db5b..08047151b 100644 --- a/server/templates/staff/jobs/calculate_slips.html +++ b/server/templates/staff/jobs/calculate_slips.html @@ -31,8 +31,10 @@

    +

    Calculate slips and save as a .csv file.

    {% call forms.render_form(form, action_text='Calculate Slips') %} {{ forms.render_field(form.timescale) }} + {{ forms.render_checkbox_field(form.show_results) }} {% endcall %}
    diff --git a/server/utils.py b/server/utils.py index 2c1d80bb6..481e1bd70 100644 --- a/server/utils.py +++ b/server/utils.py @@ -236,6 +236,16 @@ def chunks(l, n): prev_index = index +def output_csv_iterable(header, rows): + """ Generate csv string for given header list and list of rows (lists). """ + output = StringIO() + writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC) + writer.writerow(header) + [writer.writerow(row) for row in rows] + rows = output.getvalue().split('\r\n') + return [bytes(row + '\r\n', 'utf-8') for row in rows] + + def generate_csv(query, items, selector_fn): """ Generate csv export of scores for assignment. selector_fn: 1 arg function that returns a list of dictionaries From 28d916dd49663f644518233800d0f236be97f703 Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Thu, 4 May 2017 21:33:18 -0700 Subject: [PATCH 07/26] add course slips form --- server/controllers/admin.py | 40 +++++++++++++++-- server/forms.py | 11 ++++- .../staff/course/assignment/assignments.html | 22 +++++++++- .../slips.assign.html} | 0 .../staff/jobs/slips/slips.course.html | 44 +++++++++++++++++++ 5 files changed, 111 insertions(+), 6 deletions(-) rename server/templates/staff/jobs/{calculate_slips.html => slips/slips.assign.html} (100%) create mode 100644 server/templates/staff/jobs/slips/slips.course.html diff --git a/server/controllers/admin.py b/server/controllers/admin.py index 434adf0a1..af1a5a104 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -974,7 +974,7 @@ def checkpoint_grading(cid, aid): form=form, ) -@admin.route("/course//assignments//slip", +@admin.route("/course//assignments//slips", methods=["GET", "POST"]) @is_staff(course_arg='cid') def calculate_slips(cid, aid): @@ -984,7 +984,7 @@ def calculate_slips(cid, aid): flash('Cannot access assignment', 'error') return abort(404) - form = forms.SlipCalculatorForm() + form = forms.AssignSlipCalculatorForm() timescale = form.timescale.data.title() if form.validate_on_submit(): print("submitted!") @@ -1002,13 +1002,47 @@ def calculate_slips(cid, aid): return redirect(url_for('.course_job', cid=cid, job_id=job.id)) return render_template( - 'staff/jobs/calculate_slips.html', + 'staff/jobs/slips/slips.assign.html', courses=courses, current_course=current_course, assignment=assign, form=form, ) +@admin.route("/course//assignments/slips", + methods=["GET", "POST"]) +@is_staff(course_arg='cid') +def calculate_all_slips(cid): + courses, current_course = get_courses(cid) + # assgns = current_course.assignments + # active_asgns = [a for a in assgns if a.active] + # due_asgns = [a for a in assgns if not a.active] + form = forms.CourseSlipCalculatorForm() + form.assigns.choices = [('ONE', 'one'), ('TWO', 'two')] + form.process() + # timescale = form.timescale.data.title() + # if form.validate_on_submit(): + # print("submitted!") + # job = jobs.enqueue_job( + # slips.calculate_slips, + # description='Calculate Slip {} for {}'.format(timescale, assign.display_name), + # timeout=600, + # course_id=cid, + # user_id=current_user.id, + # assign_id=assign.id, + # timescale=timescale, + # show_results=form.show_results.data, + # result_kind='link', + # ) + # return redirect(url_for('.course_job', cid=cid, job_id=job.id)) + + return render_template( + 'staff/jobs/slips/slips.course.html', + courses=courses, + current_course=current_course, + form=form, + ) + ############## # Enrollment # ############## diff --git a/server/forms.py b/server/forms.py index 924fae3dd..30c3141cb 100644 --- a/server/forms.py +++ b/server/forms.py @@ -559,13 +559,22 @@ class ExportAssignment(BaseForm): description="Enable to remove identifying information from submissions") -class SlipCalculatorForm(BaseForm): +class AssignSlipCalculatorForm(BaseForm): timescale = SelectField('Time Scale', default="Days", choices=[(c, c.title()) for c in TIMESCALES], validators=[validators.required()], description="Time scale for slip calculation.") show_results = BooleanField('Show Results', default=False) +class CourseSlipCalculatorForm(BaseForm): + timescale = SelectField('Time Scale', default="Days", + choices=[(c, c.title()) for c in TIMESCALES], + validators=[validators.required()], + description="Time scale for slip calculation.") + assigns = MultiCheckboxField('Assignments', + description="Select which assignments to calculate slips for.") + show_results = BooleanField('Show Results', default=False) + ########## # Canvas # ########## diff --git a/server/templates/staff/course/assignment/assignments.html b/server/templates/staff/course/assignment/assignments.html index bdfdb9edb..ddfdea6ce 100644 --- a/server/templates/staff/course/assignment/assignments.html +++ b/server/templates/staff/course/assignment/assignments.html @@ -19,6 +19,25 @@

    {% include 'alerts.html' %} + +
    +
    +

    Actions

    +
    + +
    +
    +
    + +
    + +
    +
    @@ -66,9 +85,8 @@

    Active Assignments

    Completed Assignments

    -
    -
    diff --git a/server/templates/staff/jobs/calculate_slips.html b/server/templates/staff/jobs/slips/slips.assign.html similarity index 100% rename from server/templates/staff/jobs/calculate_slips.html rename to server/templates/staff/jobs/slips/slips.assign.html diff --git a/server/templates/staff/jobs/slips/slips.course.html b/server/templates/staff/jobs/slips/slips.course.html new file mode 100644 index 000000000..365509541 --- /dev/null +++ b/server/templates/staff/jobs/slips/slips.course.html @@ -0,0 +1,44 @@ +{% extends "staff/base.html" %} +{% import "staff/_formhelpers.html" as forms %} + +{% block title %} Calculate Slips for {{ current_course.display_name }} {% endblock %} + +{% block main %} +
    +

    + Calculate Slips for {{ current_course.display_name }} + {{ current_course.offering }} +

    +
      +
    1. + {{ current_course.offering }} +
      2. +
    2. + Assignments +
      3. +
    3. + Jobs +
      4. +
    4. + Calculate Slips +
      5. +
    +
    +
    + {% include 'alerts.html' %} +
    +
    +
    +
    +

    Calculate slips and save as a .csv file.

    + {% call forms.render_form(form, action_text='Calculate Slips') %} + {{ forms.render_field(form.timescale) }} + {{ forms.render_checkbox_field(form.assigns) }} + {{ forms.render_checkbox_field(form.show_results) }} + {% endcall %} +
    +
    +
    +
    +
    +{% endblock %} From 40736667b8327a0e3dd9f2bb00d6cbb163e4f251 Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Thu, 4 May 2017 22:17:11 -0700 Subject: [PATCH 08/26] fix validation errors --- server/controllers/admin.py | 40 ++++++++++++++++++------------------- server/forms.py | 14 ++++--------- server/jobs/slips.py | 9 ++++++++- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index af1a5a104..c6283e81e 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -987,9 +987,8 @@ def calculate_slips(cid, aid): form = forms.AssignSlipCalculatorForm() timescale = form.timescale.data.title() if form.validate_on_submit(): - print("submitted!") job = jobs.enqueue_job( - slips.calculate_slips, + slips.calculate_slips_assign, description='Calculate Slip {} for {}'.format(timescale, assign.display_name), timeout=600, course_id=cid, @@ -1014,27 +1013,26 @@ def calculate_slips(cid, aid): @is_staff(course_arg='cid') def calculate_all_slips(cid): courses, current_course = get_courses(cid) - # assgns = current_course.assignments - # active_asgns = [a for a in assgns if a.active] - # due_asgns = [a for a in assgns if not a.active] + assignments = current_course.assignments form = forms.CourseSlipCalculatorForm() - form.assigns.choices = [('ONE', 'one'), ('TWO', 'two')] + form.assigns.choices = [(a.id, a.display_name) for a in assignments] + form.assigns.default = [a.id for a in assignments] form.process() - # timescale = form.timescale.data.title() - # if form.validate_on_submit(): - # print("submitted!") - # job = jobs.enqueue_job( - # slips.calculate_slips, - # description='Calculate Slip {} for {}'.format(timescale, assign.display_name), - # timeout=600, - # course_id=cid, - # user_id=current_user.id, - # assign_id=assign.id, - # timescale=timescale, - # show_results=form.show_results.data, - # result_kind='link', - # ) - # return redirect(url_for('.course_job', cid=cid, job_id=job.id)) + timescale = form.timescale.data.title() + if form.validate_on_submit(): + job = jobs.enqueue_job( + slips.calculate_slips_course, + description="Calculate Slip {} for {}'s assignments" + .format(timescale, current_course.display_name), + timeout=600, + course_id=cid, + user_id=current_user.id, + timescale=timescale, + assigns=form.assigns.data, + show_results=form.show_results.data, + result_kind='link', + ) + return redirect(url_for('.course_job', cid=cid, job_id=job.id)) return render_template( 'staff/jobs/slips/slips.course.html', diff --git a/server/forms.py b/server/forms.py index 30c3141cb..5da7f3afc 100644 --- a/server/forms.py +++ b/server/forms.py @@ -560,20 +560,14 @@ class ExportAssignment(BaseForm): class AssignSlipCalculatorForm(BaseForm): - timescale = SelectField('Time Scale', default="Days", - choices=[(c, c.title()) for c in TIMESCALES], - validators=[validators.required()], + timescale = SelectField('Time Scale', default="days", + choices=[(c.lower(), c.title()) for c in TIMESCALES], description="Time scale for slip calculation.") show_results = BooleanField('Show Results', default=False) -class CourseSlipCalculatorForm(BaseForm): - timescale = SelectField('Time Scale', default="Days", - choices=[(c, c.title()) for c in TIMESCALES], - validators=[validators.required()], - description="Time scale for slip calculation.") - assigns = MultiCheckboxField('Assignments', +class CourseSlipCalculatorForm(AssignSlipCalculatorForm): + assigns = MultiCheckboxField('Assignments', coerce=int, description="Select which assignments to calculate slips for.") - show_results = BooleanField('Show Results', default=False) ########## # Canvas # diff --git a/server/jobs/slips.py b/server/jobs/slips.py index c28ff892f..e0929b303 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -15,7 +15,14 @@ def timediff(created, deadline, timescale): return math.ceil(secs_over / timescales[timescale.lower()]) @jobs.background_job -def calculate_slips(assign_id, timescale, show_results): +def calculate_slips_course(assigns, timescale, show_results): + logger = jobs.get_job_logger() + logger.info(assigns) + logger.info(timescale) + logger.info(show_results) + +@jobs.background_job +def calculate_slips_assign(assign_id, timescale, show_results): logger = jobs.get_job_logger() job = jobs.get_current_job() From ddd9e40fdc39872d2a3f66c5d7f07095e2890d2b Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Thu, 4 May 2017 22:19:43 -0700 Subject: [PATCH 09/26] cleaup function names --- server/controllers/admin.py | 8 ++++---- server/jobs/slips.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index c6283e81e..a4406e004 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -977,7 +977,7 @@ def checkpoint_grading(cid, aid): @admin.route("/course//assignments//slips", methods=["GET", "POST"]) @is_staff(course_arg='cid') -def calculate_slips(cid, aid): +def calculate_assign_slips(cid, aid): courses, current_course = get_courses(cid) assign = Assignment.query.filter_by(id=aid, course_id=cid).one_or_none() if not assign or not Assignment.can(assign, current_user, 'grade'): @@ -1011,7 +1011,7 @@ def calculate_slips(cid, aid): @admin.route("/course//assignments/slips", methods=["GET", "POST"]) @is_staff(course_arg='cid') -def calculate_all_slips(cid): +def calculate_course_slips(cid): courses, current_course = get_courses(cid) assignments = current_course.assignments form = forms.CourseSlipCalculatorForm() @@ -1021,8 +1021,8 @@ def calculate_all_slips(cid): timescale = form.timescale.data.title() if form.validate_on_submit(): job = jobs.enqueue_job( - slips.calculate_slips_course, - description="Calculate Slip {} for {}'s assignments" + slips.calculate_course_slips, + description="Calculate Slip {} for {}'s Assignments" .format(timescale, current_course.display_name), timeout=600, course_id=cid, diff --git a/server/jobs/slips.py b/server/jobs/slips.py index e0929b303..c54440cc2 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -15,14 +15,14 @@ def timediff(created, deadline, timescale): return math.ceil(secs_over / timescales[timescale.lower()]) @jobs.background_job -def calculate_slips_course(assigns, timescale, show_results): +def calculate_course_slips(assigns, timescale, show_results): logger = jobs.get_job_logger() logger.info(assigns) logger.info(timescale) logger.info(show_results) @jobs.background_job -def calculate_slips_assign(assign_id, timescale, show_results): +def calculate_assign_slips(assign_id, timescale, show_results): logger = jobs.get_job_logger() job = jobs.get_current_job() From 0fab8baa39f83909c0e07087f25bcd0bffa59b6f Mon Sep 17 00:00:00 2001 From: Eric Pai Date: Fri, 5 May 2017 01:26:39 -0700 Subject: [PATCH 10/26] finish course slips job --- server/controllers/admin.py | 11 ++- server/forms.py | 4 +- server/jobs/slips.py | 99 +++++++++++++------ .../staff/course/assignment/assignment.html | 2 +- .../staff/course/assignment/assignments.html | 2 +- 5 files changed, 81 insertions(+), 37 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index a4406e004..48f26c720 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -988,7 +988,7 @@ def calculate_assign_slips(cid, aid): timescale = form.timescale.data.title() if form.validate_on_submit(): job = jobs.enqueue_job( - slips.calculate_slips_assign, + slips.calculate_assign_slips, description='Calculate Slip {} for {}'.format(timescale, assign.display_name), timeout=600, course_id=cid, @@ -1014,10 +1014,13 @@ def calculate_assign_slips(cid, aid): def calculate_course_slips(cid): courses, current_course = get_courses(cid) assignments = current_course.assignments + form = forms.CourseSlipCalculatorForm() - form.assigns.choices = [(a.id, a.display_name) for a in assignments] - form.assigns.default = [a.id for a in assignments] - form.process() + inactive_assigns = [a for a in assignments if not a.active] + form.assigns.choices = [(a.id, a.display_name) for a in inactive_assigns] + form.assigns.default = [a.id for a in inactive_assigns] + form.process(request.form) + timescale = form.timescale.data.title() if form.validate_on_submit(): job = jobs.enqueue_job( diff --git a/server/forms.py b/server/forms.py index 5da7f3afc..900e81489 100644 --- a/server/forms.py +++ b/server/forms.py @@ -566,8 +566,8 @@ class AssignSlipCalculatorForm(BaseForm): show_results = BooleanField('Show Results', default=False) class CourseSlipCalculatorForm(AssignSlipCalculatorForm): - assigns = MultiCheckboxField('Assignments', coerce=int, - description="Select which assignments to calculate slips for.") + assigns = MultiCheckboxField('Completed Assignments ', coerce=int, + description="Select which completed assignments to calculate slips for.") ########## # Canvas # diff --git a/server/jobs/slips.py b/server/jobs/slips.py index c54440cc2..1fe8712be 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -1,7 +1,8 @@ -from datetime import datetime as dt import math import io import csv +from collections import defaultdict +from datetime import datetime as dt from server import jobs from server.models import Assignment, ExternalFile @@ -14,20 +15,76 @@ def timediff(created, deadline, timescale): secs_over = (created - deadline).total_seconds() return math.ceil(secs_over / timescales[timescale.lower()]) + +def save_csv(csv_name, header, rows, show_results, user, course, logger): + logger.info('Outputting csv...\n') + csv_iterable = output_csv_iterable(header, rows) + + logger.info('Uploading...') + upload = ExternalFile.upload(csv_iterable, + user_id=user.id, course_id=course.id, name=csv_name, + prefix='jobs/slips/{}'.format(course.offering)) + logger.info('Saved as: {}'.format(upload.object_name)) + + download_link = "/files/{}".format(encode_id(upload.id)) + logger.info('Download link: {} (see "result" above)\n'.format(download_link)) + + if show_results: + logger.info('Results:\n') + csv_data = ''.join([row.decode('utf-8') for row in csv_iterable]) + logger.info(csv_data) + + return download_link + + @jobs.background_job def calculate_course_slips(assigns, timescale, show_results): logger = jobs.get_job_logger() - logger.info(assigns) - logger.info(timescale) - logger.info(show_results) + logger.info('Calculating Slip {}...\n'.format(timescale.title())) + + job = jobs.get_current_job() + user = job.user + course = job.course + assigns_set = set(assigns) + assigns = (a for a in course.assignments if a.id in assigns_set) + + course_slips = defaultdict(list) + for i, assign in enumerate(assigns, 1): + logger.info('Processing {} ({} of {})...' + .format(assign.display_name, i, len(assigns_set))) + subms = assign.course_submissions(include_empty=False) + deadline = assign.due_date + assign_slips = {} + for subm in subms: + email = subm['user']['email'] + created = subm['backup']['created'] + slips = max(0, timediff(created, deadline, timescale)) + assign_slips[email] = [(assign.display_name, slips)] + course_slips = {k:course_slips[k] + assign_slips[k] + for k in course_slips.keys() | assign_slips.keys()} + + def get_row(email, assign_slips): + total_slips = sum((s for a, s in assign_slips)) + assignments = ', '.join([a for a, s in assign_slips if s > 0]) + return (email, total_slips, assignments) + + header = ( + 'User Email', + 'Slip {} Used'.format(timescale.title()), + 'Late Assignments') + rows = (get_row(*user_slips) for user_slips in course_slips.items()) + created_time = local_time(dt.now(), course, fmt='%m-%d-%I-%M-%p') + csv_name = '{}_{}.csv'.format(course.offering.replace('/', '-'), created_time) + + return save_csv(csv_name, header, rows, show_results, user, course, logger) + @jobs.background_job def calculate_assign_slips(assign_id, timescale, show_results): logger = jobs.get_job_logger() - job = jobs.get_current_job() - logger.info('Calculating Slip {}...'.format(timescale.title())) - + + user = jobs.get_current_job().user assignment = Assignment.query.get(assign_id) course = assignment.course subms = assignment.course_submissions(include_empty=False) @@ -36,30 +93,14 @@ def calculate_assign_slips(assign_id, timescale, show_results): def get_row(subm): email = subm['user']['email'] created = subm['backup']['created'] - slips = min(0, timediff(created, deadline, timescale)) - return [email, slips] + slips = max(0, timediff(created, deadline, timescale)) + return (email, slips) - header = ('User Email', 'Slip {} Used'.format(timescale.title())) + header = ( + 'User Email', + 'Slip {} Used'.format(timescale.title())) rows = (get_row(subm) for subm in subms) - logger.info('Outputting csv...\n') - csv_iterable = output_csv_iterable(header, rows) - - logger.info('Uploading...') created_time = local_time(dt.now(), course, fmt='%m-%d-%I-%M-%p') csv_name = '{}_{}.csv'.format(assignment.name.replace('/', '-'), created_time) - upload = ExternalFile.upload(csv_iterable, - user_id=job.user.id, course_id=course.id, name=csv_name, - prefix='jobs/slips/{}'.format(course.offering)) - logger.info('Saved as: {}'.format(upload.object_name)) - - download_link = "/files/{}".format(encode_id(upload.id)) - logger.info('Download link: {} (see "result" above)\n'.format(download_link)) - - if show_results: - logger.info('Results:\n') - csv_data = ''.join([row.decode('utf-8') for row in csv_iterable]) - logger.info(csv_data) - - return download_link - + return save_csv(csv_name, header, rows, show_results, user, course, logger) \ No newline at end of file diff --git a/server/templates/staff/course/assignment/assignment.html b/server/templates/staff/course/assignment/assignment.html index 3d192fe61..6f0f649d8 100644 --- a/server/templates/staff/course/assignment/assignment.html +++ b/server/templates/staff/course/assignment/assignment.html @@ -92,7 +92,7 @@

    Actions

  • Grant Extension
    • -
  • +
  • Calculate Slips
  • diff --git a/server/templates/staff/course/assignment/assignments.html b/server/templates/staff/course/assignment/assignments.html index ddfdea6ce..2a10d0b49 100644 --- a/server/templates/staff/course/assignment/assignments.html +++ b/server/templates/staff/course/assignment/assignments.html @@ -30,7 +30,7 @@

    Actions

    • From 2c15c3b2e1dbacd563b9e2c354a8ba967d5ff1a4 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Sun, 16 Feb 2020 22:15:17 -0800 Subject: [PATCH 11/26] Added the TODO list to upgrade the previous pull request --- server/jobs/slips.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 1fe8712be..a86053bb1 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -9,6 +9,18 @@ from server.utils import encode_id, local_time, output_csv_iterable from server.constants import TIMESCALES +""" + TODO: + - Use TIMESCALES instead of timescales + - Look through the code and search for optimizations + - Restrict calculation of slip days to only one assignment (weird otherwise) + - Calculate slip days separately for each relevant submission + - Remove the "show results" button + - Show SID in addition to the email + - Work around output_csv_iterable +""" + + timescales = {'days':86400, 'hours':3600, 'minutes':60} def timediff(created, deadline, timescale): From f804a16959a21f46463961c28fe617e4b24d9060 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Tue, 18 Feb 2020 12:35:18 -0800 Subject: [PATCH 12/26] Added user ID to csm --- server/jobs/slips.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index a86053bb1..1b1871195 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -16,6 +16,7 @@ - Restrict calculation of slip days to only one assignment (weird otherwise) - Calculate slip days separately for each relevant submission - Remove the "show results" button + "show results" displays the table in the logger. Should we keep it? - Show SID in addition to the email - Work around output_csv_iterable """ @@ -90,7 +91,10 @@ def get_row(email, assign_slips): return save_csv(csv_name, header, rows, show_results, user, course, logger) - +""" +Changed this method so that it also returns the user's id. +Is subm['user']['id'] the same as SID? +""" @jobs.background_job def calculate_assign_slips(assign_id, timescale, show_results): logger = jobs.get_job_logger() @@ -103,12 +107,14 @@ def calculate_assign_slips(assign_id, timescale, show_results): deadline = assignment.due_date def get_row(subm): + sid = subm['user']['id'] email = subm['user']['email'] created = subm['backup']['created'] slips = max(0, timediff(created, deadline, timescale)) - return (email, slips) + return sid, email, slips header = ( + 'User SID', 'User Email', 'Slip {} Used'.format(timescale.title())) rows = (get_row(subm) for subm in subms) From 099f50f06f58f71267134fc03ca4ef5c931b5e27 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 19 Feb 2020 00:53:05 -0800 Subject: [PATCH 13/26] Specified questions --- server/jobs/slips.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 1b1871195..cc4de6d2e 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -49,6 +49,14 @@ def save_csv(csv_name, header, rows, show_results, user, course, logger): return download_link +""" +Of how many submissions is the user's score comprised? +Which submissions do we consider relevant? How many relevant submissions can there be? +Would a pair of final submission and revision backup be sufficient? +""" +def return_relevant_submissions(assignment, user): + pass + @jobs.background_job def calculate_course_slips(assigns, timescale, show_results): From 77945cbd2299eb1660cf73900f50f3eeb2471753 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Mon, 24 Feb 2020 02:22:25 -0800 Subject: [PATCH 14/26] Added SID to calculate slips on a single assignment --- server/jobs/slips.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index cc4de6d2e..e67c20adc 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -5,7 +5,7 @@ from datetime import datetime as dt from server import jobs -from server.models import Assignment, ExternalFile +from server.models import Assignment, ExternalFile, User from server.utils import encode_id, local_time, output_csv_iterable from server.constants import TIMESCALES @@ -17,7 +17,7 @@ - Calculate slip days separately for each relevant submission - Remove the "show results" button "show results" displays the table in the logger. Should we keep it? - - Show SID in addition to the email + - Show SID in addition to the email (NOT THE ID!) - Work around output_csv_iterable """ @@ -53,6 +53,10 @@ def save_csv(csv_name, header, rows, show_results, user, course, logger): Of how many submissions is the user's score comprised? Which submissions do we consider relevant? How many relevant submissions can there be? Would a pair of final submission and revision backup be sufficient? + +Points, Style points, Check points +Points -> give best submission for effort, total and regrade + """ def return_relevant_submissions(assignment, user): pass @@ -115,7 +119,10 @@ def calculate_assign_slips(assign_id, timescale, show_results): deadline = assignment.due_date def get_row(subm): - sid = subm['user']['id'] + user_id = subm['user']['id'] + user = User.get_by_id(user_id) + enrollment = user.enrollments()[0] + sid = enrollment.sid email = subm['user']['email'] created = subm['backup']['created'] slips = max(0, timediff(created, deadline, timescale)) From fd2abe63108f2a345ae5dbc0557d9a34d3425d90 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Mon, 24 Feb 2020 03:03:33 -0800 Subject: [PATCH 15/26] Modified calculate_assign_slips to calculate slips for the FINAL submission --- server/jobs/slips.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index e67c20adc..f21eeef96 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -15,10 +15,8 @@ - Look through the code and search for optimizations - Restrict calculation of slip days to only one assignment (weird otherwise) - Calculate slip days separately for each relevant submission - - Remove the "show results" button - "show results" displays the table in the logger. Should we keep it? - - Show SID in addition to the email (NOT THE ID!) - Work around output_csv_iterable + - Redesign calculating slips for the whole course """ @@ -103,10 +101,21 @@ def get_row(email, assign_slips): return save_csv(csv_name, header, rows, show_results, user, course, logger) -""" -Changed this method so that it also returns the user's id. -Is subm['user']['id'] the same as SID? -""" + +def get_students_with_submission(assignment): + """Get a list of IDs of students who have made a submission + for the current assignment. + + :param ASSIGNMENT instance of the model Assignment + + This code is copied from the assignment_stats() method + in the Assignment model methods. May need refactoring.""" + + data = assignment.course_submissions() + students_ids = set(s['user']['id'] for s in data if s['backup'] and s['backup']['submit']) + return students_ids + + @jobs.background_job def calculate_assign_slips(assign_id, timescale, show_results): logger = jobs.get_job_logger() @@ -115,16 +124,16 @@ def calculate_assign_slips(assign_id, timescale, show_results): user = jobs.get_current_job().user assignment = Assignment.query.get(assign_id) course = assignment.course - subms = assignment.course_submissions(include_empty=False) + students_ids = get_students_with_submission(assignment) + subms = [assignment.final_submission([id]) for id in students_ids] deadline = assignment.due_date def get_row(subm): - user_id = subm['user']['id'] - user = User.get_by_id(user_id) - enrollment = user.enrollments()[0] + curr_user = subm.submitter + enrollment = curr_user.enrollments()[0] sid = enrollment.sid - email = subm['user']['email'] - created = subm['backup']['created'] + email = curr_user.email + created = subm.submission_time slips = max(0, timediff(created, deadline, timescale)) return sid, email, slips From 7e22a9e21f01143674bde10f020ee6352d765d06 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 26 Feb 2020 01:08:25 -0800 Subject: [PATCH 16/26] Tested on the database, calculate_assign_slips functions correctly --- server/jobs/slips.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index f21eeef96..d80669917 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -56,8 +56,6 @@ def save_csv(csv_name, header, rows, show_results, user, course, logger): Points -> give best submission for effort, total and regrade """ -def return_relevant_submissions(assignment, user): - pass @jobs.background_job From 785beb7a2f819366d31226dfb60ae3bdc7f48811 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Tue, 3 Mar 2020 23:24:44 -0800 Subject: [PATCH 17/26] Changed filename --- server/jobs/slips.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index d80669917..674e0f31c 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -34,7 +34,7 @@ def save_csv(csv_name, header, rows, show_results, user, course, logger): logger.info('Uploading...') upload = ExternalFile.upload(csv_iterable, user_id=user.id, course_id=course.id, name=csv_name, - prefix='jobs/slips/{}'.format(course.offering)) + prefix='slips_') logger.info('Saved as: {}'.format(upload.object_name)) download_link = "/files/{}".format(encode_id(upload.id)) @@ -140,7 +140,7 @@ def get_row(subm): 'User Email', 'Slip {} Used'.format(timescale.title())) rows = (get_row(subm) for subm in subms) - created_time = local_time(dt.now(), course, fmt='%m-%d-%I-%M-%p') - csv_name = '{}_{}.csv'.format(assignment.name.replace('/', '-'), created_time) + created_time = local_time(dt.now(), course, fmt='%m-%d_%I-%M-%p') + csv_name = '{}_{}.csv'.format(assignment.display_name.replace('/', '-'), created_time) return save_csv(csv_name, header, rows, show_results, user, course, logger) \ No newline at end of file From 97301ee40248e03c9dfa493fbdeaf04393814ea9 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Tue, 3 Mar 2020 23:45:27 -0800 Subject: [PATCH 18/26] Fixed slips calculation for the whole course --- server/jobs/slips.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 674e0f31c..4053fca42 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -68,32 +68,29 @@ def calculate_course_slips(assigns, timescale, show_results): course = job.course assigns_set = set(assigns) assigns = (a for a in course.assignments if a.id in assigns_set) + rows = [] - course_slips = defaultdict(list) for i, assign in enumerate(assigns, 1): logger.info('Processing {} ({} of {})...' - .format(assign.display_name, i, len(assigns_set))) - subms = assign.course_submissions(include_empty=False) + .format(assign.display_name, i, len(assigns_set))) + students_ids = get_students_with_submission(assign) + subms = [assign.final_submission([id]) for id in students_ids] deadline = assign.due_date - assign_slips = {} for subm in subms: - email = subm['user']['email'] - created = subm['backup']['created'] + curr_user = subm.submitter + enrollment = curr_user.enrollments()[0] + sid = enrollment.sid + email = curr_user.email + created = subm.submission_time slips = max(0, timediff(created, deadline, timescale)) - assign_slips[email] = [(assign.display_name, slips)] - course_slips = {k:course_slips[k] + assign_slips[k] - for k in course_slips.keys() | assign_slips.keys()} - - def get_row(email, assign_slips): - total_slips = sum((s for a, s in assign_slips)) - assignments = ', '.join([a for a, s in assign_slips if s > 0]) - return (email, total_slips, assignments) - + if slips > 0: + rows.append((assign.display_name, sid, email, slips)) + header = ( - 'User Email', - 'Slip {} Used'.format(timescale.title()), - 'Late Assignments') - rows = (get_row(*user_slips) for user_slips in course_slips.items()) + 'Assignment', + 'User SID', + 'User Email', + 'Slip {} Used'.format(timescale.title())) created_time = local_time(dt.now(), course, fmt='%m-%d-%I-%M-%p') csv_name = '{}_{}.csv'.format(course.offering.replace('/', '-'), created_time) From a2d6a086564f204d151309903d5e44dc630c06f9 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 4 Mar 2020 00:19:47 -0800 Subject: [PATCH 19/26] Refined slips jobs --- server/jobs/slips.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 4053fca42..b0e2f299f 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -12,15 +12,14 @@ """ TODO: - Use TIMESCALES instead of timescales - - Look through the code and search for optimizations - - Restrict calculation of slip days to only one assignment (weird otherwise) - - Calculate slip days separately for each relevant submission - Work around output_csv_iterable - - Redesign calculating slips for the whole course + + - Support for timezone? """ -timescales = {'days':86400, 'hours':3600, 'minutes':60} +timescales = {'days': 86400, 'hours': 3600, 'minutes': 60} + def timediff(created, deadline, timescale): secs_over = (created - deadline).total_seconds() @@ -83,6 +82,7 @@ def calculate_course_slips(assigns, timescale, show_results): email = curr_user.email created = subm.submission_time slips = max(0, timediff(created, deadline, timescale)) + logger.info('LOG: ' + email + ' ' + str(slips)) if slips > 0: rows.append((assign.display_name, sid, email, slips)) @@ -91,8 +91,8 @@ def calculate_course_slips(assigns, timescale, show_results): 'User SID', 'User Email', 'Slip {} Used'.format(timescale.title())) - created_time = local_time(dt.now(), course, fmt='%m-%d-%I-%M-%p') - csv_name = '{}_{}.csv'.format(course.offering.replace('/', '-'), created_time) + created_time = local_time(dt.now(), course, fmt='%m-%d_%I-%M-%p') + csv_name = '{}_{}.csv'.format(course.display_name.replace('/', '-'), created_time) return save_csv(csv_name, header, rows, show_results, user, course, logger) @@ -122,21 +122,22 @@ def calculate_assign_slips(assign_id, timescale, show_results): students_ids = get_students_with_submission(assignment) subms = [assignment.final_submission([id]) for id in students_ids] deadline = assignment.due_date + rows = [] - def get_row(subm): + for subm in subms: curr_user = subm.submitter enrollment = curr_user.enrollments()[0] sid = enrollment.sid email = curr_user.email created = subm.submission_time slips = max(0, timediff(created, deadline, timescale)) - return sid, email, slips + if slips > 0: + rows.append((sid, email, slips)) header = ( 'User SID', 'User Email', 'Slip {} Used'.format(timescale.title())) - rows = (get_row(subm) for subm in subms) created_time = local_time(dt.now(), course, fmt='%m-%d_%I-%M-%p') csv_name = '{}_{}.csv'.format(assignment.display_name.replace('/', '-'), created_time) From 6bd8c63ee04904a6d3b33f579667808334db78dd Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 4 Mar 2020 00:46:56 -0800 Subject: [PATCH 20/26] Got rid of output_csv_iterable --- server/jobs/slips.py | 40 ++++++++++++++++++---------------------- server/utils.py | 11 ----------- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index b0e2f299f..79969d6f1 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -6,15 +6,12 @@ from server import jobs from server.models import Assignment, ExternalFile, User -from server.utils import encode_id, local_time, output_csv_iterable +from server.utils import encode_id, local_time, generate_csv from server.constants import TIMESCALES """ TODO: - - Use TIMESCALES instead of timescales - - Work around output_csv_iterable - - - Support for timezone? + - Support for timezone in filename? """ @@ -28,7 +25,16 @@ def timediff(created, deadline, timescale): def save_csv(csv_name, header, rows, show_results, user, course, logger): logger.info('Outputting csv...\n') - csv_iterable = output_csv_iterable(header, rows) + + def selector_fn(lst): + if len(lst) != len(header): + raise IndexError + result = {} + for i in range(len(lst)): + result[header[i]] = lst[i] + return [result] + + csv_iterable = list(map(lambda x: bytes(x, 'utf-8'), generate_csv(rows, header, selector_fn))) logger.info('Uploading...') upload = ExternalFile.upload(csv_iterable, @@ -46,16 +52,6 @@ def save_csv(csv_name, header, rows, show_results, user, course, logger): return download_link -""" -Of how many submissions is the user's score comprised? -Which submissions do we consider relevant? How many relevant submissions can there be? -Would a pair of final submission and revision backup be sufficient? - -Points, Style points, Check points -Points -> give best submission for effort, total and regrade - -""" - @jobs.background_job def calculate_course_slips(assigns, timescale, show_results): @@ -84,13 +80,13 @@ def calculate_course_slips(assigns, timescale, show_results): slips = max(0, timediff(created, deadline, timescale)) logger.info('LOG: ' + email + ' ' + str(slips)) if slips > 0: - rows.append((assign.display_name, sid, email, slips)) + rows.append([assign.display_name, sid, email, slips]) - header = ( + header = [ 'Assignment', 'User SID', 'User Email', - 'Slip {} Used'.format(timescale.title())) + 'Slip {} Used'.format(timescale.title())] created_time = local_time(dt.now(), course, fmt='%m-%d_%I-%M-%p') csv_name = '{}_{}.csv'.format(course.display_name.replace('/', '-'), created_time) @@ -132,12 +128,12 @@ def calculate_assign_slips(assign_id, timescale, show_results): created = subm.submission_time slips = max(0, timediff(created, deadline, timescale)) if slips > 0: - rows.append((sid, email, slips)) + rows.append([sid, email, slips]) - header = ( + header = [ 'User SID', 'User Email', - 'Slip {} Used'.format(timescale.title())) + 'Slip {} Used'.format(timescale.title())] created_time = local_time(dt.now(), course, fmt='%m-%d_%I-%M-%p') csv_name = '{}_{}.csv'.format(assignment.display_name.replace('/', '-'), created_time) diff --git a/server/utils.py b/server/utils.py index 12bade9a6..5269ce35d 100644 --- a/server/utils.py +++ b/server/utils.py @@ -256,17 +256,6 @@ def chunks(l, n): yield l[prev_index:index] prev_index = index - -def output_csv_iterable(header, rows): - """ Generate csv string for given header list and list of rows (lists). """ - output = StringIO() - writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC) - writer.writerow(header) - [writer.writerow(row) for row in rows] - rows = output.getvalue().split('\r\n') - return [bytes(row + '\r\n', 'utf-8') for row in rows] - - def generate_csv(query, items, selector_fn): """ Generate csv export of scores for assignment. selector_fn: 1 arg function that returns a list of dictionaries From c3a8cd5057affc3291a7b570e5560bb1aa7d8e7c Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 4 Mar 2020 01:02:38 -0800 Subject: [PATCH 21/26] Fixed bug when user hasn't submitted yet --- server/jobs/slips.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 79969d6f1..f2cf5d899 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -69,7 +69,11 @@ def calculate_course_slips(assigns, timescale, show_results): logger.info('Processing {} ({} of {})...' .format(assign.display_name, i, len(assigns_set))) students_ids = get_students_with_submission(assign) - subms = [assign.final_submission([id]) for id in students_ids] + subms = [] + for id in students_ids: + subm = assign.final_submission([id]) + if subm: + subms.append(subm) deadline = assign.due_date for subm in subms: curr_user = subm.submitter @@ -78,7 +82,6 @@ def calculate_course_slips(assigns, timescale, show_results): email = curr_user.email created = subm.submission_time slips = max(0, timediff(created, deadline, timescale)) - logger.info('LOG: ' + email + ' ' + str(slips)) if slips > 0: rows.append([assign.display_name, sid, email, slips]) @@ -116,10 +119,13 @@ def calculate_assign_slips(assign_id, timescale, show_results): assignment = Assignment.query.get(assign_id) course = assignment.course students_ids = get_students_with_submission(assignment) - subms = [assignment.final_submission([id]) for id in students_ids] + subms = [] + for id in students_ids: + subm = assignment.final_submission([id]) + if subm: + subms.append(subm) deadline = assignment.due_date rows = [] - for subm in subms: curr_user = subm.submitter enrollment = curr_user.enrollments()[0] From 9a85b24ee18532cfe6a243f4aec11b19ba5dbaa0 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 4 Mar 2020 01:04:30 -0800 Subject: [PATCH 22/26] Specified TODO --- server/jobs/slips.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index f2cf5d899..7f8c327cd 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -12,6 +12,7 @@ """ TODO: - Support for timezone in filename? + - Remake templates as specified in the old pull request? """ From bf606774916e8a0edb0ecc1cbcf60d4338ac317a Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 4 Mar 2020 13:57:17 -0800 Subject: [PATCH 23/26] Changed TIMESCALES --- server/constants.py | 4 +++- server/forms.py | 2 +- server/jobs/slips.py | 5 +---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server/constants.py b/server/constants.py index 45022dd04..1d84de1ce 100644 --- a/server/constants.py +++ b/server/constants.py @@ -1,5 +1,6 @@ """App constants""" import os +from collections import OrderedDict STUDENT_ROLE = 'student' GRADER_ROLE = 'grader' @@ -21,7 +22,8 @@ 'regrade', 'revision', 'checkpoint 1', 'checkpoint 2', 'private', 'autograder', 'error'] -TIMESCALES = ['days', 'hours', 'minutes'] +#TIMESCALES = ['days', 'hours', 'minutes'] +TIMESCALES = OrderedDict([('days', 86400), ('hours', 3600), ('minutes', 60)]) API_PREFIX = '/api' OAUTH_SCOPES = ['all', 'email'] diff --git a/server/forms.py b/server/forms.py index bf3650136..757badbf2 100644 --- a/server/forms.py +++ b/server/forms.py @@ -772,7 +772,7 @@ class ExportAssignment(BaseForm): class AssignSlipCalculatorForm(BaseForm): timescale = SelectField('Time Scale', default="days", - choices=[(c.lower(), c.title()) for c in TIMESCALES], + choices=[(c.lower(), c.title()) for c in TIMESCALES.keys()], description="Time scale for slip calculation.") show_results = BooleanField('Show Results', default=False) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 7f8c327cd..7f6d20216 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -16,12 +16,9 @@ """ -timescales = {'days': 86400, 'hours': 3600, 'minutes': 60} - - def timediff(created, deadline, timescale): secs_over = (created - deadline).total_seconds() - return math.ceil(secs_over / timescales[timescale.lower()]) + return math.ceil(secs_over / TIMESCALES[timescale.lower()]) def save_csv(csv_name, header, rows, show_results, user, course, logger): From 23d95032d9226dc6861604e82ac7aa47ff46311b Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Wed, 4 Mar 2020 13:58:02 -0800 Subject: [PATCH 24/26] Removed commented old TIMESCALES --- server/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/constants.py b/server/constants.py index 1d84de1ce..a16f7ca56 100644 --- a/server/constants.py +++ b/server/constants.py @@ -22,7 +22,6 @@ 'regrade', 'revision', 'checkpoint 1', 'checkpoint 2', 'private', 'autograder', 'error'] -#TIMESCALES = ['days', 'hours', 'minutes'] TIMESCALES = OrderedDict([('days', 86400), ('hours', 3600), ('minutes', 60)]) API_PREFIX = '/api' From da4b3d62e3faec448ee2922c3b4aae8dc0753504 Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Tue, 24 Mar 2020 04:44:05 -0700 Subject: [PATCH 25/26] Reworked course slips --- server/jobs/slips.py | 76 +++++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 7f6d20216..18934548a 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -26,7 +26,7 @@ def save_csv(csv_name, header, rows, show_results, user, course, logger): def selector_fn(lst): if len(lst) != len(header): - raise IndexError + raise IndexError(str(lst) + " " + str(header)) result = {} for i in range(len(lst)): result[header[i]] = lst[i] @@ -60,34 +60,66 @@ def calculate_course_slips(assigns, timescale, show_results): user = job.user course = job.course assigns_set = set(assigns) - assigns = (a for a in course.assignments if a.id in assigns_set) + assigns = [a for a in course.assignments if a.id in assigns_set] rows = [] - for i, assign in enumerate(assigns, 1): - logger.info('Processing {} ({} of {})...' - .format(assign.display_name, i, len(assigns_set))) - students_ids = get_students_with_submission(assign) - subms = [] - for id in students_ids: - subm = assign.final_submission([id]) + # for i, assign in enumerate(assignments, 1): + # logger.info('Processing {} ({} of {})...' + # .format(assign.display_name, i, len(assigns_set))) + # students_ids = get_students_with_submission(assign) + # subms = [] + # for id in students_ids: + # subm = assign.final_submission([id]) + # if subm: + # subms.append(subm) + # deadline = assign.due_date + # for subm in subms: + # curr_user = subm.submitter + # enrollment = curr_user.enrollments()[0] + # sid = enrollment.sid + # email = curr_user.email + # created = subm.submission_time + # slips = max(0, timediff(created, deadline, timescale)) + # if slips > 0: + # rows.append([assign.display_name, sid, email, slips]) + + enrollments = job.course.get_students() + for enrollment in enrollments: + sid = enrollment.sid + student = enrollment.user + email = student.email + row = [sid, email] + student_id = student.id + logger.info('Processing {}\'s submissions'.format(email)) + for assignment in assigns: + deadline = assignment.due_date + subm = assignment.final_submission([student_id]) if subm: - subms.append(subm) - deadline = assign.due_date - for subm in subms: - curr_user = subm.submitter - enrollment = curr_user.enrollments()[0] - sid = enrollment.sid - email = curr_user.email - created = subm.submission_time - slips = max(0, timediff(created, deadline, timescale)) - if slips > 0: - rows.append([assign.display_name, sid, email, slips]) + created = subm.submission_time + slips = max(0, timediff(created, deadline, timescale)) + else: + slips = 0 + row.append(slips) + rows.append(row) + + + + + # header = [ + # 'Assignment', + # 'User SID', + # 'User Email', + # 'Slip {} Used'.format(timescale.title())] header = [ - 'Assignment', 'User SID', 'User Email', - 'Slip {} Used'.format(timescale.title())] + ] + for assignment in assigns: + assign_name = assignment.display_name + header.append('Slip {} Used on '.format(timescale.title()) + + assign_name) + created_time = local_time(dt.now(), course, fmt='%m-%d_%I-%M-%p') csv_name = '{}_{}.csv'.format(course.display_name.replace('/', '-'), created_time) From d53da93260d1640c09cdf8f6f091c02d7593c5bd Mon Sep 17 00:00:00 2001 From: Evgenii Sizykh Date: Tue, 24 Mar 2020 05:23:13 -0700 Subject: [PATCH 26/26] Removed comments --- server/jobs/slips.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/server/jobs/slips.py b/server/jobs/slips.py index 18934548a..ef8f8b087 100644 --- a/server/jobs/slips.py +++ b/server/jobs/slips.py @@ -63,26 +63,6 @@ def calculate_course_slips(assigns, timescale, show_results): assigns = [a for a in course.assignments if a.id in assigns_set] rows = [] - # for i, assign in enumerate(assignments, 1): - # logger.info('Processing {} ({} of {})...' - # .format(assign.display_name, i, len(assigns_set))) - # students_ids = get_students_with_submission(assign) - # subms = [] - # for id in students_ids: - # subm = assign.final_submission([id]) - # if subm: - # subms.append(subm) - # deadline = assign.due_date - # for subm in subms: - # curr_user = subm.submitter - # enrollment = curr_user.enrollments()[0] - # sid = enrollment.sid - # email = curr_user.email - # created = subm.submission_time - # slips = max(0, timediff(created, deadline, timescale)) - # if slips > 0: - # rows.append([assign.display_name, sid, email, slips]) - enrollments = job.course.get_students() for enrollment in enrollments: sid = enrollment.sid @@ -102,15 +82,6 @@ def calculate_course_slips(assigns, timescale, show_results): row.append(slips) rows.append(row) - - - - # header = [ - # 'Assignment', - # 'User SID', - # 'User Email', - # 'Slip {} Used'.format(timescale.title())] - header = [ 'User SID', 'User Email',