Benchmark Team Size Mutations

.gitmodules

@@ -3,4 +3,4 @@
     url = https://github.com/Teamable-Analytics/algorithms-simulation-cache.git
 [submodule "api/ai/external_algorithms/group_matcher_algorithm/group-matcher"]
 	path = api/ai/external_algorithms/group_matcher_algorithm/group-matcher
-	url = git@github.com:ketphan02/group-matcher.git
+	url = https://github.com/ketphan02/group-matcher.git

api/ai/priority_algorithm/mutations/team_size_low_disruption.py

@@ -19,9 +19,9 @@ class TeamSizeLowDisruptionMutation(Mutation):
     This minimizes the change in teams and hopefully won't tear apart teams that are doing well.
     """
 
-    def __init__(self, num_teams: int = 2, *args, **kwargs):
+    def __init__(self, number_of_teams: int = 2, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.num_teams = num_teams
+        self.num_teams = number_of_teams
 
     def mutate_one(
         self,

benchmarking/data/simulated_data/realistic_class/providers.py

@@ -1,3 +1,4 @@
+import itertools
 from typing import List
 
 import numpy as np
@@ -11,7 +12,12 @@
 )
 from api.dataclasses.project import Project, ProjectRequirement
 from api.dataclasses.student import Student
-from benchmarking.data.interfaces import StudentProvider
+from api.dataclasses.team import TeamShell
+from benchmarking.data.interfaces import StudentProvider, InitialTeamsProvider
+from benchmarking.data.simulated_data.mock_initial_teams_provider import (
+    MockInitialTeamsProvider,
+    MockInitialTeamsProviderSettings,
+)
 from benchmarking.data.simulated_data.mock_student_provider import (
     MockStudentProvider,
     MockStudentProviderSettings,
@@ -92,16 +98,40 @@ def get(self, seed: int = None) -> List[Student]:
                         (4, 0.05),
                     ],
                     GITHUB_EXPERIENCE: [
-                        (GithubExperience.Beginner, 0.2),
-                        (GithubExperience.Intermediate, 0.7),
-                        (GithubExperience.Advanced, 0.1),
+                        (
+                            GithubExperience.Beginner,
+                            0.2,
+                        ),
+                        (
+                            GithubExperience.Intermediate,
+                            0.7,
+                        ),
+                        (
+                            GithubExperience.Advanced,
+                            0.1,
+                        ),
                     ],
                     WORK_EXPERIENCE: [
-                        (WorkExperience.No_Experience, 0.4),
-                        (WorkExperience.One_Semester, 0.05),
-                        (WorkExperience.Two_Semesters, 0.2),
-                        (WorkExperience.Three_Semesters, 0.3),
-                        (WorkExperience.More_Than_Three_Semesters, 0.05),
+                        (
+                            WorkExperience.No_Experience,
+                            0.4,
+                        ),
+                        (
+                            WorkExperience.One_Semester,
+                            0.05,
+                        ),
+                        (
+                            WorkExperience.Two_Semesters,
+                            0.2,
+                        ),
+                        (
+                            WorkExperience.Three_Semesters,
+                            0.3,
+                        ),
+                        (
+                            WorkExperience.More_Than_Three_Semesters,
+                            0.05,
+                        ),
                     ],
                 },
             )
@@ -132,6 +162,29 @@ def get(self, seed: int = None) -> List[Student]:
         return [students[_] for _ in order]
 
 
+class RealisticMockInitialTeamsProvider(InitialTeamsProvider):
+    def __init__(self, num_teams: int):
+        self.num_teams = num_teams
+
+    def get(self) -> List[TeamShell]:
+        projects = []
+        project_cycler = itertools.cycle(get_realistic_projects())
+        for i in range(self.num_teams):
+            next_project = next(project_cycler)
+            projects.append(
+                Project(
+                    _id=i,
+                    name=next_project.name + " " + str(i),
+                    requirements=next_project.requirements,
+                )
+            )
+        return MockInitialTeamsProvider(
+            settings=MockInitialTeamsProviderSettings(
+                projects=projects,
+            )
+        ).get()
+
+
 def get_realistic_projects() -> List[Project]:
     """
     A list of five projects with realistic requirements.

benchmarking/runs/mutations/mutation_benchmarking.py

@@ -0,0 +1,208 @@
+from typing import Dict
+
+import typer
+
+from api.ai.interfaces.algorithm_config import PriorityAlgorithmConfig
+from api.ai.priority_algorithm.mutations.greedy_local_max import GreedyLocalMaxMutation
+from api.ai.priority_algorithm.mutations.local_max import LocalMaxMutation
+from api.ai.priority_algorithm.mutations.local_max_random import LocalMaxRandomMutation
+from api.ai.priority_algorithm.mutations.random_slice import RandomSliceMutation
+from api.ai.priority_algorithm.mutations.random_swap import RandomSwapMutation
+from api.ai.priority_algorithm.mutations.robinhood import RobinhoodMutation
+from api.ai.priority_algorithm.mutations.robinhood_holistic import (
+    RobinhoodHolisticMutation,
+)
+from api.dataclasses.enums import ScenarioAttribute, Gender, Race, AlgorithmType
+from benchmarking.data.simulated_data.realistic_class.providers import (
+    RealisticMockInitialTeamsProvider,
+    RealisticMockStudentProvider,
+)
+from benchmarking.evaluations.graphing.graph_metadata import GraphData, GraphAxisRange
+from benchmarking.evaluations.graphing.line_graph import line_graph
+from benchmarking.evaluations.graphing.line_graph_metadata import LineGraphMetadata
+from benchmarking.evaluations.metrics.average_project_requirements_coverage import (
+    AverageProjectRequirementsCoverage,
+)
+from benchmarking.evaluations.metrics.average_solo_status import AverageSoloStatus
+from benchmarking.evaluations.metrics.cosine_similarity import AverageCosineDifference
+from benchmarking.evaluations.metrics.priority_satisfaction import PrioritySatisfaction
+from benchmarking.evaluations.scenarios.satisfy_project_requirements_and_diversify_female_min_of_2_and_diversify_african_min_of_2 import (
+    SatisfyProjectRequirementsAndDiversifyFemaleMinOf2AndDiversifyAfricanMinOf2,
+)
+from benchmarking.runs.interfaces import Run
+from benchmarking.simulation.goal_to_priority import goals_to_priorities
+from benchmarking.simulation.insight import Insight
+from benchmarking.simulation.simulation_set import SimulationSetArtifact, SimulationSet
+from benchmarking.simulation.simulation_settings import SimulationSettings
+
+
+class MutationBenchmarking(Run):
+    def start(self, num_trials: int = 100, generate_graphs: bool = False):
+        class_sizes = [40, 100, 240, 500, 1000]
+        team_size = 5
+
+        scenario = (
+            SatisfyProjectRequirementsAndDiversifyFemaleMinOf2AndDiversifyAfricanMinOf2()
+        )
+
+        metrics = {
+            "PrioritySatisfaction": PrioritySatisfaction(
+                goals_to_priorities(scenario.goals),
+                False,
+            ),
+            "AverageProjectRequirementsCoverage": AverageProjectRequirementsCoverage(),
+            "AverageCosineDifferenceGender": AverageCosineDifference(
+                attribute_filter=[ScenarioAttribute.GENDER.value],
+                name="AverageCosineDifferenceGender",
+            ),
+            "AverageCosineDifferenceRace": AverageCosineDifference(
+                attribute_filter=[ScenarioAttribute.RACE.value],
+                name="AverageCosineDifferenceRace",
+            ),
+            "AverageSoloStatus": AverageSoloStatus(
+                minority_groups_map={
+                    ScenarioAttribute.GENDER.value: [Gender.FEMALE.value],
+                    ScenarioAttribute.RACE.value: [Race.African.value],
+                },
+            ),
+            Insight.KEY_RUNTIMES: "runtime",
+        }
+
+        max_keep = 30
+        max_spread = 100
+        max_iterate = 250
+        max_time = 1_000_000
+        mutation_sets = {
+            "mutate_random": [RandomSwapMutation(max_spread)],
+            "mutate_local_max": [
+                LocalMaxMutation(num_mutations=1),
+                RandomSwapMutation(num_mutations=max_spread - 1),
+            ],
+            "mutate_local_max_random": [
+                LocalMaxRandomMutation(num_mutations=5),
+                RandomSwapMutation(num_mutations=(max_spread - 5)),
+            ],
+            "mutate_random_slice": [RandomSliceMutation(max_spread)],
+            "mutate_half_random_slice": [
+                RandomSliceMutation(num_mutations=(max_spread // 2)),
+                RandomSwapMutation(num_mutations=(max_spread // 2)),
+            ],
+            "mutate_greedy_local_max_n_2": [
+                GreedyLocalMaxMutation(number_of_teams=2, num_mutations=max_spread)
+            ],
+            "mutate_greedy_local_max_n_4": [
+                GreedyLocalMaxMutation(number_of_teams=4, num_mutations=max_spread)
+            ],
+            "mutate_greedy_local_max_n_8": [
+                GreedyLocalMaxMutation(number_of_teams=8, num_mutations=max_spread)
+            ],
+            "mutate_greedy_local_max_with_random_swap": [
+                GreedyLocalMaxMutation(num_mutations=(max_spread // 2)),
+                RandomSwapMutation(num_mutations=(max_spread // 2)),
+            ],
+            "mutate_greedy_local_max_with_random_slice_n_2": [
+                GreedyLocalMaxMutation(num_mutations=(max_spread // 2)),
+                RandomSliceMutation(num_mutations=(max_spread // 2)),
+            ],
+            "mutate_greedy_local_max_with_random_slice_n_4": [
+                GreedyLocalMaxMutation(
+                    number_of_teams=4, num_mutations=(max_spread // 2)
+                ),
+                RandomSliceMutation(max_spread // 2),
+            ],
+            "mutate_greedy_local_max_with_random_slice_n_8": [
+                GreedyLocalMaxMutation(
+                    number_of_teams=8, num_mutations=(max_spread // 2)
+                ),
+                (RandomSliceMutation(), max_spread // 2),
+            ],
+            "mutate_robinhood": [RobinhoodMutation(max_spread)],
+            "mutate_robinhood_half_random_swap": [
+                RobinhoodMutation(max_spread // 2),
+                RandomSwapMutation(max_spread // 2),
+            ],
+            "mutate_robinhood_holistic": [RobinhoodHolisticMutation(max_spread)],
+            "mutate_robinhood_holistic_half_random_swap": [
+                RobinhoodHolisticMutation(max_spread // 2),
+                RandomSwapMutation(max_spread // 2),
+            ],
+        }
+
+        artifacts: Dict[int, Dict[str, SimulationSetArtifact]] = {}
+
+        for class_size in class_sizes:
+            if class_size not in artifacts:
+                artifacts[class_size] = {}
+            for mutation_name, mutations in mutation_sets.items():
+                artifacts[class_size][mutation_name] = SimulationSet(
+                    settings=SimulationSettings(
+                        scenario=scenario,
+                        student_provider=RealisticMockStudentProvider(class_size),
+                        cache_key=f"mutations/mutation_benchmarking/{mutation_name}/class_size_{class_size}/",
+                        initial_teams_provider=RealisticMockInitialTeamsProvider(
+                            class_size // team_size
+                        ),
+                    ),
+                    algorithm_set={
+                        AlgorithmType.PRIORITY: [
+                            PriorityAlgorithmConfig(
+                                MAX_KEEP=max_keep,
+                                MAX_SPREAD=max_spread,
+                                MAX_ITERATE=max_iterate,
+                                MAX_TIME=max_time,
+                                MUTATIONS=mutations,
+                            ),
+                        ]
+                    },
+                ).run(num_runs=num_trials)
+
+        if generate_graphs:
+            for metric_name, metric in metrics.items():
+                data = {}
+                for class_size, _ in artifacts.items():
+                    if class_size not in data:
+                        data[class_size] = {}
+                    for mutation_name, artifact in _.items():
+                        insight_output_set = Insight.get_output_set(
+                            artifact=artifact,
+                            metrics=[
+                                metric
+                                if metric != "runtime"
+                                else AverageProjectRequirementsCoverage()
+                            ],
+                        )
+                        avg_metric = list(
+                            Insight.average_metric(
+                                insight_output_set=insight_output_set,
+                                metric_name=metric_name
+                                if metric_name == Insight.KEY_RUNTIMES
+                                else metric.name,
+                            ).values()
+                        )[0]
+                        data[class_size][mutation_name] = avg_metric
+
+                graph_data = [
+                    GraphData(
+                        x_data=class_sizes,
+                        y_data=[
+                            data[class_size][mutation_name]
+                            for class_size in class_sizes
+                        ],
+                        name=mutation_name,
+                    )
+                    for mutation_name in mutation_sets.keys()
+                ]
+
+                line_graph(
+                    LineGraphMetadata(
+                        x_label="Class size",
+                        y_label=metric_name,
+                        title=f"Class Size vs. {metric_name}",
+                        data=graph_data,
+                        y_lim=GraphAxisRange(0, 1),
+                    )
+                )
+
+
+if __name__ == "__main__":
+    typer.run(MutationBenchmarking().start)