From 0b553231abeea3b84fa345980546ca37ae4a927a Mon Sep 17 00:00:00 2001
From: Gertjan van den Burg <gertjanvandenburg@gmail.com>
Date: Tue, 27 Aug 2019 10:34:09 +0100
Subject: Score user on demo and ask to redo if subpar

---
 app/main/demo.py | 46 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/main/demo.py b/app/main/demo.py
index 2c0f1bc..42525b1 100644
--- a/app/main/demo.py
+++ b/app/main/demo.py
@@ -290,6 +290,31 @@ DEMO_DATA = {
 }
 
 
+def demo_performance(user_id):
+    score = 0
+    for demo_id in DEMO_DATA:
+        dataset = Dataset.query.filter_by(
+            name=DEMO_DATA[demo_id]["dataset"]["name"]
+        ).first()
+        task = Task.query.filter_by(
+            annotator_id=user_id, dataset_id=dataset.id
+        ).first()
+        annotations = (
+            Annotation.query.join(Task, Annotation.task)
+            .filter_by(task_id=task.id)
+            .all()
+        )
+        true_cp = get_demo_true_cps(dataset.name)
+        user_cp = [a.cp_index for a in annotations]
+        if len(true_cp) == len(user_cp) == 0:
+            score += 1
+            continue
+
+        n_correct, n_window, n_fp = metrics(true_cp, user_cp)
+        score += (n_correct + n_window - n_fp) / len(true_cp)
+    return score / len(DEMO_DATA)
+
+
 def redirect_user(demo_id, phase_id):
     last_demo_id = max(DEMO_DATA.keys())
     demo_last_phase_id = 3
@@ -298,6 +323,17 @@ def redirect_user(demo_id, phase_id):
         if current_user.is_introduced:
             return redirect(url_for("main.index"))
 
+        # check user performance
+        if demo_performance(current_user.id) < 0.75:
+            flash(
+                "Unfortunately your performance on the introduction "
+                "datasets was not as high as we would like. Please go "
+                "through the introduction one more time to make sure "
+                "that you understand and are comfortable with change "
+                "point detection."
+            )
+            return redirect(url_for("main.index"))
+
         # mark user as introduced
         current_user.is_introduced = True
         db.session.commit()
@@ -382,8 +418,8 @@ def process_annotations(demo_id):
     return retval
 
 
-def get_user_feedback(true_cp, user_cp):
-    """Generate HTML to show as feedback to the user"""
+def metrics(true_cp, user_cp):
+    true_cp = [int(x) for x in true_cp]
     user_cp = [int(x) for x in user_cp]
 
     correct = []
@@ -413,6 +449,12 @@ def get_user_feedback(true_cp, user_cp):
     n_correct = len(correct)
     n_window = len(window)
     n_fp = len(incorrect)
+    return n_correct, n_window, n_fp
+
+
+def get_user_feedback(true_cp, user_cp):
+    """Generate HTML to show as feedback to the user"""
+    n_correct, n_window, n_fp = metrics(true_cp, user_cp)
 
     text = "\n\n*Feedback:*\n\n"
     if len(true_cp) == len(user_cp) == 0:
-- 
cgit v1.2.3