diff --git a/README.md b/README.md index 7377f98..9ab7113 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,21 @@ This will run the asset pipeline, start the Python server, and start BrowserSync To add new dependencies, add them to `requirements.in` and then run `pip-compile requirements.in` to produce a new locked `requirements.txt`. Do not edit `requirements.txt` directly as it will be overwritten by future PRs. +### Profiling + +To profile routes to figure out how long it's taking and what's taking the most time, in `conditional/__init__.py` add + +```python +from werkzeug.middleware.profiler import ProfilerMiddleware + +app.wsgi_app = ProfilerMiddleware( + app.wsgi_app, + restrictions=[30] +) +``` + +after app initialization + ### Database Migrations If the database schema is changed after initializing the database, you must migrate it to the new schema by running: diff --git a/conditional/blueprints/dashboard.py b/conditional/blueprints/dashboard.py index 527e8dd..a0a5c86 100644 --- a/conditional/blueprints/dashboard.py +++ b/conditional/blueprints/dashboard.py @@ -36,6 +36,7 @@ def display_dashboard(user_dict=None): log.info('display dashboard') # Get the list of voting members. + can_vote = get_voting_members() data = {} @@ -56,7 +57,7 @@ def display_dashboard(user_dict=None): spring = {} c_meetings = get_cm(user_dict['account']) spring['committee_meetings'] = len(c_meetings) - spring['req_meetings'] = req_cm(user_dict['account']) + spring['req_meetings'] = req_cm(user_dict['account'].uid) h_meetings = [(m.meeting_id, m.attendance_status) for m in get_hm(user_dict['account'])] spring['hm_missed'] = len([h for h in h_meetings if h[1] == "Absent"]) eval_entry = SpringEval.query.filter(SpringEval.uid == user_dict['account'].uid, diff --git a/conditional/blueprints/intro_evals.py b/conditional/blueprints/intro_evals.py index 3349bcb..cfda1e2 100644 --- a/conditional/blueprints/intro_evals.py +++ b/conditional/blueprints/intro_evals.py @@ -2,6 +2,7 @@ import structlog from flask import Blueprint, request +from sqlalchemy import func from conditional import start_of_year, auth from conditional.models.models import CommitteeMeeting @@ -11,82 +12,109 @@ from conditional.models.models import FreshmanHouseMeetingAttendance from conditional.models.models import FreshmanSeminarAttendance from conditional.models.models import HouseMeeting +from conditional.models.models import MemberCommitteeAttendance from conditional.models.models import MemberHouseMeetingAttendance from conditional.models.models import MemberSeminarAttendance from conditional.models.models import TechnicalSeminar from conditional.util.auth import get_user from conditional.util.flask import render_template from conditional.util.ldap import ldap_get_intro_members -from conditional.util.member import get_cm, get_hm intro_evals_bp = Blueprint('intro_evals_bp', __name__) logger = structlog.get_logger() - -@intro_evals_bp.route('/intro_evals/') -@auth.oidc_auth("default") -@get_user -def display_intro_evals(internal=False, user_dict=None): - log = logger.new(request=request, auth_dict=user_dict) - log.info('Display Intro Evals Listing') - - # get user data - def get_fid_cm_count(member_id): - return len([a for a in FreshmanCommitteeAttendance.query.filter( - FreshmanCommitteeAttendance.fid == member_id) - if CommitteeMeeting.query.filter(CommitteeMeeting.id == a.meeting_id).first().approved]) - - members = ldap_get_intro_members() - - ie_members = [] +def get_intro_members_without_accounts(): + freshman_cm_count = dict([tuple(row) for row in FreshmanCommitteeAttendance.query.join( + CommitteeMeeting, + FreshmanCommitteeAttendance.meeting_id == CommitteeMeeting.id + ).with_entities( + FreshmanCommitteeAttendance.fid, + CommitteeMeeting.timestamp, + CommitteeMeeting.approved, + ).filter( + CommitteeMeeting.approved, + CommitteeMeeting.timestamp >= start_of_year() + ).with_entities( + FreshmanCommitteeAttendance.fid, + func.count(FreshmanCommitteeAttendance.fid) #pylint: disable=not-callable + ).group_by( + FreshmanCommitteeAttendance.fid + ).all()]) + + freshman_hm_missed = dict([tuple(row) for row in FreshmanHouseMeetingAttendance.query.join( + HouseMeeting, + FreshmanHouseMeetingAttendance.meeting_id == HouseMeeting.id + ).filter( + HouseMeeting.date >= start_of_year(), + FreshmanHouseMeetingAttendance.attendance_status == 'Absent' + ).with_entities( + FreshmanHouseMeetingAttendance.fid, + func.count(FreshmanHouseMeetingAttendance.fid) #pylint: disable=not-callable + ).group_by( + FreshmanHouseMeetingAttendance.fid + ).all()]) + + freshman_ts_attendance_query = FreshmanSeminarAttendance.query.join( + TechnicalSeminar, + FreshmanSeminarAttendance.seminar_id == TechnicalSeminar.id + ).with_entities( + FreshmanSeminarAttendance.fid, + TechnicalSeminar.timestamp, + TechnicalSeminar.approved + ).filter( + TechnicalSeminar.approved, + TechnicalSeminar.timestamp >= start_of_year() + ).with_entities( + FreshmanSeminarAttendance.fid, + TechnicalSeminar.name + ).all() + + freshman_ts_attendance_dict = {} + + for row in freshman_ts_attendance_query: + if not row[0] in freshman_ts_attendance_dict: + freshman_ts_attendance_dict[row[0]] = [] + + freshman_ts_attendance_dict[row[0]].append(row[1]) # freshmen who don't have accounts - fids = list(FreshmanAccount.query.filter( + freshman_accounts = list(FreshmanAccount.query.filter( FreshmanAccount.eval_date > start_of_year(), FreshmanAccount.eval_date > datetime.now())) - for fid in fids: - h_meetings = [m.meeting_id for m in - FreshmanHouseMeetingAttendance.query.filter( - FreshmanHouseMeetingAttendance.fid == fid.id - ).filter( - FreshmanHouseMeetingAttendance.attendance_status == "Absent" - )] + ie_members = [] - if fid.signatures_missed is None: + for freshman_account in freshman_accounts: + missed_hms = [] + if freshman_hm_missed.get(freshman_account.id, 0) != 0: + missed_hms = FreshmanHouseMeetingAttendance.query.join( + HouseMeeting, + FreshmanHouseMeetingAttendance.meeting_id == HouseMeeting.id + ).filter( + HouseMeeting.date >= start_of_year(), # TODO: this needs to be fixed + FreshmanHouseMeetingAttendance.attendance_status == 'Absent', + FreshmanHouseMeetingAttendance.fid == freshman_account.id, + ).with_entities( + func.array_agg(HouseMeeting.date) + ).scalar() + + if freshman_account.signatures_missed is None: signatures_missed = -1 else: - signatures_missed = fid.signatures_missed + signatures_missed = freshman_account.signatures_missed + + cms_attended = freshman_cm_count.get(freshman_account.id, 0) freshman = { - 'name': fid.name, - 'uid': fid.id, - 'eval_date': fid.eval_date.strftime("%Y-%m-%d"), + 'name': freshman_account.name, + 'uid': freshman_account.id, + 'eval_date': freshman_account.eval_date.strftime("%Y-%m-%d"), 'signatures_missed': signatures_missed, - 'committee_meetings': get_fid_cm_count(fid.id), - 'committee_meetings_passed': get_fid_cm_count(fid.id) >= 6, - 'house_meetings_missed': - [ - { - "date": m.date.strftime("%Y-%m-%d"), - "reason": - FreshmanHouseMeetingAttendance.query.filter( - FreshmanHouseMeetingAttendance.fid == fid.id).filter( - FreshmanHouseMeetingAttendance.meeting_id == m.id).first().excuse - } - for m in HouseMeeting.query.filter( - HouseMeeting.id.in_(h_meetings) - ) - ], - 'technical_seminars': - [s.name for s in TechnicalSeminar.query.filter( - TechnicalSeminar.id.in_( - [a.seminar_id for a in FreshmanSeminarAttendance.query.filter( - FreshmanSeminarAttendance.fid == fid.id) - if TechnicalSeminar.query.filter(TechnicalSeminar.id == a.seminar_id).first().approved] - )) - ], + 'committee_meetings': cms_attended, + 'committee_meetings_passed': cms_attended >= 6, + 'house_meetings_missed': missed_hms, + 'technical_seminars': freshman_ts_attendance_dict.get(freshman_account.id, []), 'social_events': '', 'comments': "", 'ldap_account': False, @@ -94,6 +122,72 @@ def get_fid_cm_count(member_id): } ie_members.append(freshman) + return ie_members + +@intro_evals_bp.route('/intro_evals/') +@auth.oidc_auth("default") +@get_user +def display_intro_evals(internal=False, user_dict=None): + log = logger.new(request=request, auth_dict=user_dict) + log.info('Display Intro Evals Listing') + + members = ldap_get_intro_members() + + ie_members = get_intro_members_without_accounts() + + account_cm_count = dict([tuple(row) for row in MemberCommitteeAttendance.query.join( + CommitteeMeeting, + MemberCommitteeAttendance.meeting_id == CommitteeMeeting.id + ).with_entities( + MemberCommitteeAttendance.uid, + CommitteeMeeting.timestamp, + CommitteeMeeting.approved, + ).filter( + CommitteeMeeting.approved, + CommitteeMeeting.timestamp >= start_of_year() + ).with_entities( + MemberCommitteeAttendance.uid, + func.count(MemberCommitteeAttendance.uid) #pylint: disable=not-callable + ).group_by( + MemberCommitteeAttendance.uid + ).all()]) + + account_hm_missed = dict([tuple(row) for row in MemberHouseMeetingAttendance.query.join( + HouseMeeting, + MemberHouseMeetingAttendance.meeting_id == HouseMeeting.id + ).filter( + HouseMeeting.date >= start_of_year(), + MemberHouseMeetingAttendance.attendance_status == 'Absent' + ).with_entities( + MemberHouseMeetingAttendance.uid, + func.count(MemberHouseMeetingAttendance.uid) #pylint: disable=not-callable + ).group_by( + MemberHouseMeetingAttendance.uid + ).all()]) + + account_ts_attendance_query = MemberSeminarAttendance.query.join( + TechnicalSeminar, + MemberSeminarAttendance.seminar_id == TechnicalSeminar.id + ).with_entities( + MemberSeminarAttendance.uid, + TechnicalSeminar.timestamp, + TechnicalSeminar.approved + ).filter( + TechnicalSeminar.approved, + TechnicalSeminar.timestamp >= start_of_year() + ).with_entities( + MemberSeminarAttendance.uid, + TechnicalSeminar.name + ).all() + + account_ts_attendance_dict = {} + + for row in account_ts_attendance_query: + if not row[0] in account_ts_attendance_dict: + account_ts_attendance_dict[row[0]] = [] + + account_ts_attendance_dict[row[0]].append(row[1]) + # freshmen who have accounts for member in members: uid = member.uid @@ -107,38 +201,31 @@ def get_fid_cm_count(member_id): if freshman_data.freshman_eval_result != "Pending" and internal: continue - h_meetings = [m.meeting_id for m in get_hm(member, only_absent=True)] + member_missed_hms = [] + + if account_hm_missed.get(uid, 0) != 0: + member_missed_hms = MemberHouseMeetingAttendance.query.join( + HouseMeeting, + MemberHouseMeetingAttendance.meeting_id == HouseMeeting.id + ).filter( + HouseMeeting.date >= start_of_year(), + MemberHouseMeetingAttendance.attendance_status == 'Absent', + MemberHouseMeetingAttendance.uid == uid, + ).with_entities( + func.array_agg(HouseMeeting.date) + ).scalar() + + cms_attended = account_cm_count.get(uid, 0) + member_info = { 'name': name, 'uid': uid, 'eval_date': freshman_data.eval_date.strftime("%Y-%m-%d"), 'signatures_missed': freshman_data.signatures_missed, - 'committee_meetings': len(get_cm(member)), - 'committee_meetings_passed': len(get_cm(member)) >= 6, - 'house_meetings_missed': - [ - { - "date": m.date.strftime("%Y-%m-%d"), - "reason": - MemberHouseMeetingAttendance.query.filter( - MemberHouseMeetingAttendance.uid == uid, - MemberHouseMeetingAttendance.meeting_id == m.id).first().excuse - } - for m in HouseMeeting.query.filter( - HouseMeeting.id.in_(h_meetings) - ) - ], - 'technical_seminars': - [seminar.name for seminar in TechnicalSeminar.query.join( - MemberSeminarAttendance, - MemberSeminarAttendance.seminar_id == TechnicalSeminar.id - ).with_entities( - TechnicalSeminar.name - ).filter( - TechnicalSeminar.timestamp > start_of_year(), - MemberSeminarAttendance.uid == member.uid, - TechnicalSeminar.approved == True # pylint: disable=singleton-comparison - ).all()], + 'committee_meetings': cms_attended, + 'committee_meetings_passed': cms_attended >= 6, + 'house_meetings_missed': member_missed_hms, + 'technical_seminars': account_ts_attendance_dict.get(uid, []), 'social_events': freshman_data.social_events, 'comments': freshman_data.other_notes, 'ldap_account': True, diff --git a/conditional/blueprints/spring_evals.py b/conditional/blueprints/spring_evals.py index 212ca88..80c9194 100644 --- a/conditional/blueprints/spring_evals.py +++ b/conditional/blueprints/spring_evals.py @@ -1,15 +1,14 @@ import structlog from flask import Blueprint, request +from sqlalchemy import func from conditional import db, start_of_year, auth -from conditional.models.models import HouseMeeting -from conditional.models.models import MajorProject -from conditional.models.models import MemberHouseMeetingAttendance -from conditional.models.models import SpringEval +from conditional.models.models import CommitteeMeeting, CurrentCoops, HouseMeeting, MemberCommitteeAttendance +from conditional.models.models import MajorProject, MemberHouseMeetingAttendance, SpringEval from conditional.util.auth import get_user from conditional.util.flask import render_template from conditional.util.ldap import ldap_get_active_members -from conditional.util.member import get_cm, get_hm, req_cm +from conditional.util.member import req_cm spring_evals_bp = Blueprint('spring_evals_bp', __name__) @@ -25,77 +24,115 @@ def display_spring_evals(internal=False, user_dict=None): active_members = ldap_get_active_members() + cm_count = dict([tuple(row) for row in MemberCommitteeAttendance.query.join( + CommitteeMeeting, + MemberCommitteeAttendance.meeting_id == CommitteeMeeting.id + ).with_entities( + MemberCommitteeAttendance.uid, + CommitteeMeeting.timestamp, + CommitteeMeeting.approved, + ).filter( + CommitteeMeeting.approved, + CommitteeMeeting.timestamp >= start_of_year() + ).with_entities( + MemberCommitteeAttendance.uid, + func.count(MemberCommitteeAttendance.uid) #pylint: disable=not-callable + ).group_by( + MemberCommitteeAttendance.uid + ).all()]) + + hm_missed = dict([tuple(row) for row in MemberHouseMeetingAttendance.query.join( + HouseMeeting, + MemberHouseMeetingAttendance.meeting_id == HouseMeeting.id + ).filter( + HouseMeeting.date >= start_of_year(), + MemberHouseMeetingAttendance.attendance_status == 'Absent' + ).with_entities( + MemberHouseMeetingAttendance.uid, + func.count(MemberHouseMeetingAttendance.uid) #pylint: disable=not-callable + ).group_by( + MemberHouseMeetingAttendance.uid + ).all()]) + + major_project_query = MajorProject.query.filter( + MajorProject.date >= start_of_year() + ).all() + + coop_members = CurrentCoops.query.filter( + CurrentCoops.date_created >= start_of_year() + ).with_entities( + func.array_agg(CurrentCoops.uid) + ).scalar() + + major_projects = {} + + for project in major_project_query: + if not project.uid in major_projects: + major_projects[project.uid] = [] + + major_projects.get(project.uid).append({ + 'name': project.name, + 'status': project.status, + 'description': project.description + }) + sp_members = [] for account in active_members: uid = account.uid + name = account.cn + spring_entry = SpringEval.query.filter( - SpringEval.date_created > start_of_year(), + SpringEval.date_created >= start_of_year(), SpringEval.uid == uid, - SpringEval.active == True).first() # pylint: disable=singleton-comparison + SpringEval.active).first() # pylint: disable=singleton-comparison if spring_entry is None: spring_entry = SpringEval(uid) db.session.add(spring_entry) db.session.flush() db.session.commit() - elif spring_entry.status != "Pending" and internal: + elif spring_entry.status != 'Pending' and internal: continue - eval_data = None + member_missed_hms = [] + + if hm_missed.get(uid, 0) != 0: + member_missed_hms = MemberHouseMeetingAttendance.query.join( + HouseMeeting, + MemberHouseMeetingAttendance.meeting_id == HouseMeeting.id + ).filter( + HouseMeeting.date >= start_of_year(), + MemberHouseMeetingAttendance.attendance_status == 'Absent', + MemberHouseMeetingAttendance.uid == uid, + ).with_entities( + func.array_agg(HouseMeeting.date) + ).scalar() + + cm_attended_count = cm_count.get(uid, 0) + member_major_projects = major_projects.get(uid, []) + + passed_mps = [project for project in member_major_projects if project['status'] == 'Passed'] - h_meetings = [m.meeting_id for m in get_hm(account, only_absent=True)] member = { - 'name': account.cn, + 'name': name, 'uid': uid, 'status': spring_entry.status, - 'committee_meetings': len(get_cm(account)), - 'req_meetings': req_cm(account), - 'house_meetings_missed': - [ - { - "date": m.date.strftime("%Y-%m-%d"), - "reason": - MemberHouseMeetingAttendance.query.filter( - MemberHouseMeetingAttendance.uid == uid).filter( - MemberHouseMeetingAttendance.meeting_id == m.id).first().excuse - } - for m in HouseMeeting.query.filter( - HouseMeeting.id.in_(h_meetings) - ) - ], - 'major_projects': [ - { - 'name': p.name, - 'status': p.status, - 'description': p.description - } for p in MajorProject.query.filter( - MajorProject.date > start_of_year(), - MajorProject.uid == uid)] + 'committee_meetings': cm_attended_count, + 'req_meetings': req_cm(uid, coop_members), + 'house_meetings_missed': member_missed_hms, + 'major_projects': member_major_projects } + member['major_projects_len'] = len(member['major_projects']) - member['major_projects_passed'] = [ - { - 'name': p.name, - 'status': p.status, - 'description': p.description - } for p in MajorProject.query.filter( - MajorProject.date > start_of_year(), - MajorProject.status == "Passed", - MajorProject.uid == uid)] + member['major_projects_passed'] = passed_mps member['major_projects_passed_len'] = len(member['major_projects_passed']) - member['major_project_passed'] = False - for mp in member['major_projects']: - if mp['status'] == "Passed": - member['major_project_passed'] = True - break - - if internal: - member['housing_evals'] = eval_data + member['major_project_passed'] = member['major_projects_passed_len'] > 0 + sp_members.append(member) sp_members.sort(key=lambda x: x['committee_meetings'], reverse=True) sp_members.sort(key=lambda x: len(x['house_meetings_missed'])) - sp_members.sort(key=lambda x: len([p for p in x['major_projects'] if p['status'] == "Passed"]), reverse=True) + sp_members.sort(key=lambda x: x['major_projects_passed_len'], reverse=True) # return names in 'first last (username)' format if internal: return sp_members diff --git a/conditional/templates/spring_evals.html b/conditional/templates/spring_evals.html index 8f4e4e8..6b9d265 100644 --- a/conditional/templates/spring_evals.html +++ b/conditional/templates/spring_evals.html @@ -114,8 +114,8 @@