Experiments

Running the experiment

This is an example of an experimental study based on solving three problems of the ZDT family with three different algorithms: NSGA-II, GDE3 and SMPSO.

The hypervolume, generational distance and epsilon indicators are used for performance assessment.

from jmetal.algorithm.multiobjective.gde3 import GDE3
from jmetal.algorithm.multiobjective.nsgaii import NSGAII
from jmetal.algorithm.multiobjective.smpso import SMPSO
from jmetal.core.quality_indicator import *
from jmetal.lab.experiment import Experiment, Job, generate_summary_from_experiment
from jmetal.operator import PolynomialMutation, SBXCrossover
from jmetal.problem import ZDT1, ZDT2, ZDT3
from jmetal.util.archive import CrowdingDistanceArchive
from jmetal.util.termination_criterion import StoppingByEvaluations


def configure_experiment(problems: dict, n_run: int):
    jobs = []
    max_evaluations = 25000

    for run in range(n_run):
        for problem_tag, problem in problems.items():
            jobs.append(
                Job(
                    algorithm=NSGAII(
                        problem=problem,
                        population_size=100,
                        offspring_population_size=100,
                        mutation=PolynomialMutation(probability=1.0 / problem.number_of_variables(),
                                                    distribution_index=20),
                        crossover=SBXCrossover(probability=1.0, distribution_index=20),
                        termination_criterion=StoppingByEvaluations(max_evaluations=max_evaluations)
                    ),
                    algorithm_tag='NSGAII',
                    problem_tag=problem_tag,
                    run=run,
                )
            )
            jobs.append(
                Job(
                    algorithm=GDE3(
                        problem=problem,
                        population_size=100,
                        cr=0.5,
                        f=0.5,
                        termination_criterion=StoppingByEvaluations(max_evaluations=max_evaluations)
                    ),
                    algorithm_tag='GDE3',
                    problem_tag=problem_tag,
                    run=run,
                )
            )
            jobs.append(
                Job(
                    algorithm=SMPSO(
                        problem=problem,
                        swarm_size=100,
                        mutation=PolynomialMutation(probability=1.0 / problem.number_of_variables(),
                                                    distribution_index=20),
                        leaders=CrowdingDistanceArchive(100),
                        termination_criterion=StoppingByEvaluations(max_evaluations=max_evaluations)
                    ),
                    algorithm_tag='SMPSO',
                    problem_tag=problem_tag,
                    run=run,
                )
            )

    return jobs


if __name__ == '__main__':
    # Configure the experiments
    jobs = configure_experiment(problems={'ZDT1': ZDT1(), 'ZDT2': ZDT2(), 'ZDT3': ZDT3()}, n_run=31)

    # Run the study
    output_directory = 'data'
    experiment = Experiment(output_dir=output_directory, jobs=jobs)
    experiment.run()

Summary file

The results of this experiment can be summarized to a CSV file as follows:

if __name__ == '__main__':
    # experiment = ...

    # Generate summary file
    generate_summary_from_experiment(
        input_dir=output_directory,
        reference_fronts='resources/reference_fronts',
        quality_indicators=[GenerationalDistance(), EpsilonIndicator(), HyperVolume([1.0, 1.0])]
    )

This file contains all the information of the quality indicator values, for each configuration and run. The summary file is the input of all the statistical tests, so that they can be applied to any valid file having the proper format.

$ head QualityIndicatorSummary.csv
Algorithm,Problem,ExecutionId,IndicatorName,IndicatorValue
NSGAII,ZDT1,0,EP,0.015705992620067832
NSGAII,ZDT1,1,EP,0.012832504015918067
...

API

experiment

logger = get_logger(__name__) module-attribute

.. module:: laboratory :platform: Unix, Windows :synopsis: Run experiments. WIP!

.. moduleauthor:: Antonio Benítez-Hidalgo antonio.b@uma.es

Experiment(output_dir, jobs, m_workers=6)

Run an experiment to execute a list of jobs.

:param output_dir: Base directory where each job will save its results. :param jobs: List of Jobs (from mod:jmetal.util.laboratory)) to be executed. :param m_workers: Maximum number of workers to execute the Jobs in parallel.

Source code in src/jmetal/lab/experiment.py

def __init__(self, output_dir: str, jobs: list[Job], m_workers: int = 6):
    """Run an experiment to execute a list of jobs.

    :param output_dir: Base directory where each job will save its results.
    :param jobs: List of Jobs (from :py:mod:`jmetal.util.laboratory)`) to be executed.
    :param m_workers: Maximum number of workers to execute the Jobs in parallel.
    """
    self.jobs = jobs
    self.m_workers = m_workers
    self.output_dir = output_dir
    self.job_data: list[Any] = []

generate_summary_from_experiment(input_dir, quality_indicators, reference_fronts='')

Compute a list of quality indicators. The input data directory must met the following structure (this is generated automatically by the Experiment class):

• algorithm_a

  • problem_a

  • FUN.0.tsv

  • FUN.1.tsv
  • VAR.0.tsv
  • VAR.1.tsv
  • ...

:param input_dir: Directory where all the input data is found (function values and variables). :param reference_fronts: Directory where reference fronts are found. :param quality_indicators: List of quality indicators to compute. :return: None.

Source code in src/jmetal/lab/experiment.py

def generate_summary_from_experiment(
    input_dir: str, quality_indicators: list[QualityIndicator], reference_fronts: str = ""
):
    """Compute a list of quality indicators. The input data directory *must* met the following structure (this is generated
    automatically by the Experiment class):

    * <base_dir>

      * algorithm_a

        * problem_a

          * FUN.0.tsv
          * FUN.1.tsv
          * VAR.0.tsv
          * VAR.1.tsv
          * ...

    :param input_dir: Directory where all the input data is found (function values and variables).
    :param reference_fronts: Directory where reference fronts are found.
    :param quality_indicators: List of quality indicators to compute.
    :return: None.
    """

    if not quality_indicators:
        quality_indicators = []

    with open("QualityIndicatorSummary.csv", "w+") as of:
        of.write("Algorithm,Problem,ExecutionId,IndicatorName,IndicatorValue\n")

    for dirname, _, filenames in os.walk(input_dir):
        for filename in filenames:
            try:
                # Linux filesystem
                algorithm, problem = dirname.split("/")[-2:]
            except ValueError:
                # Windows filesystem
                algorithm, problem = dirname.split("\\")[-2:]

            if "TIME" in filename:
                run_tag = [s for s in filename.split(".") if s.isdigit()].pop()

                with open(os.path.join(dirname, filename)) as content_file:
                    content = content_file.read()

                with open("QualityIndicatorSummary.csv", "a+") as of:
                    of.write(",".join([algorithm, problem, run_tag, "Time", str(content)]))
                    of.write("\n")

            if "FUN" in filename:
                solutions = read_solutions(os.path.join(dirname, filename))
                run_tag = [s for s in filename.split(".") if s.isdigit()].pop()
                for indicator in quality_indicators:
                    reference_front_file = os.path.join(reference_fronts, problem + ".pf")

                    # Add reference front if any
                    if hasattr(indicator, "reference_front"):
                        if Path(reference_front_file).is_file():
                            reference_front = []
                            with open(reference_front_file) as file:
                                for line in file:
                                    reference_front.append([float(x) for x in line.split()])

                            indicator.reference_front = np.array(reference_front)
                        else:
                            logger.warning("Reference front not found at", reference_front_file)

                    result = indicator.compute(
                        np.array([solutions[i].objectives for i in range(len(solutions))])
                    )

                    # Save quality indicator value to file
                    with open("QualityIndicatorSummary.csv", "a+") as of:
                        of.write(
                            ",".join(
                                [
                                    algorithm,
                                    problem,
                                    run_tag,
                                    indicator.get_short_name(),
                                    str(result),
                                ]
                            )
                        )
                        of.write("\n")

generate_median_and_wilcoxon_latex_tables(filename, output_dir='latex/meansAndWilcoxon')

Generate Latex tables including medians and IQRs. Additionally, the last algorithm is considered as the reference algorithm, and the cells include a symbol indicating whether the differences with the reference algorithm are significant or not according to the Wilcoxon rank sum test.

:param filename: Input filename (summary). :param output_dir: Output path.

Source code in src/jmetal/lab/experiment.py

def generate_median_and_wilcoxon_latex_tables(
    filename: str, output_dir: str = "latex/meansAndWilcoxon"
):
    """Generate Latex tables including medians and IQRs. Additionally, the last algorithm is considered as the reference
        algorithm, and the cells include a symbol indicating whether the differences with the reference algorithm
        are significant or not according to the Wilcoxon rank sum test.

    :param filename: Input filename (summary).
    :param output_dir: Output path.
    """
    data = pd.read_csv(filename, skipinitialspace=True)

    if len(set(data.columns.tolist())) != 5:
        raise Exception("Wrong number of columns")

    if Path(output_dir).is_dir():
        logger.warning(f"Directory {output_dir} exists. Removing contents.")
        for file in os.listdir(output_dir):
            os.remove(f"{output_dir}/{file}")
    else:
        logger.warning(f"Directory {output_dir} does not exist. Creating it.")
        Path(output_dir).mkdir(parents=True)

    algorithms = pd.unique(data["Algorithm"])
    problems = pd.unique(data["Problem"])
    indicators = pd.unique(data["IndicatorName"])

    control_algorithm = algorithms[-1]

    # Compute medians and IQRs
    medians = data.groupby(["Algorithm", "Problem", "IndicatorName"])["IndicatorValue"].median()
    iqrs = data.groupby(["Algorithm", "Problem", "IndicatorName"])["IndicatorValue"].apply(
        lambda x: iqr(x)
    )

    # Create data frame to store the Wilcoxon test results
    wilcoxon_data = pd.DataFrame(
        columns=["Indicator", "Algorithm", "Problem", "PValue", "Median", "TestResult"]
    )

    for indicator in indicators:
        for algorithm in algorithms:
            for problem in problems:
                algorithm_data = data[
                    (data["Problem"] == problem)
                    & (data["Algorithm"] == algorithm)
                    & (data["IndicatorName"] == indicator)
                ]
                ref_data = data[
                    (data["Problem"] == problem)
                    & (data["Algorithm"] == control_algorithm)
                    & (data["IndicatorName"] == indicator)
                ]
                stat, p_value = mannwhitneyu(
                    algorithm_data["IndicatorValue"], ref_data["IndicatorValue"]
                )

                test_result = ""
                if p_value <= 0.05:
                    if check_minimization(indicator):
                        if (
                            medians[algorithm][problem][indicator]
                            <= medians[control_algorithm][problem][indicator]
                        ):
                            test_result = "+"
                        else:
                            test_result = "-"
                    else:
                        if (
                            medians[algorithm][problem][indicator]
                            >= medians[control_algorithm][problem][indicator]
                        ):
                            test_result = "+"
                        else:
                            test_result = "-"
                else:
                    test_result = "="

                new_row = {
                    "Indicator": indicator,
                    "Algorithm": algorithm,
                    "Problem": problem,
                    "PValue": p_value,
                    "Median": medians[algorithm][problem][indicator],
                    "IQR": iqrs[algorithm][problem][indicator],
                    "TestResult": test_result,
                }
                wilcoxon_data = wilcoxon_data._append(new_row, ignore_index=True)

    # Generate LaTeX tables
    caption = (
        "Median and interquartile range (IQR) of the results of the {} quality indicator. "
        + "Cells with dark and light gray background highlights, respectively, the best and second best indicator values. "
        + "The algorithm in the last column is the reference "
        + "algorithm, and the symbols $+$, $-$ and $\\approx$ indicate that the differences with the reference "
        + "algorithm are significantly better, worse, or there is no difference according to the Wilcoxon rank "
        + r"sum test (confidence level: 95\%)."
    )
    for indicator_name in indicators:
        with open(
            os.path.join(output_dir, f"MedianIQRWilcoxon-{indicator_name}.tex"), "w"
        ) as latex:
            latex.write(
                __median_wilcoxon_to_latex(
                    indicator_name,
                    wilcoxon_data,
                    caption=caption.format(indicator_name),
                    label=f"table:{indicator_name}",
                )
            )

generate_kolmogorov_smirnov_latex_tables(filename, output_dir='latex/KolmogorovSmirnov')

Generate Latex tables with the results of the Kolmogorov-Smirnov test. The last algorithm is considered as the reference algorithm, and the cells include a symbol with the p-value < 0.05.

:param filename: Input filename (summary). :param output_dir: Output path.

Source code in src/jmetal/lab/experiment.py

def generate_kolmogorov_smirnov_latex_tables(
    filename: str, output_dir: str = "latex/KolmogorovSmirnov"
):
    """Generate Latex tables with the results of the Kolmogorov-Smirnov test. The last algorithm is considered as
        the reference algorithm, and the cells include a symbol with the p-value < 0.05.

    :param filename: Input filename (summary).
    :param output_dir: Output path.
    """
    data = pd.read_csv(filename, skipinitialspace=True)

    if len(set(data.columns.tolist())) != 5:
        raise Exception("Wrong number of columns")

    if Path(output_dir).is_dir():
        logger.warning(f"Directory {output_dir} exists. Removing contents.")
        for file in os.listdir(output_dir):
            os.remove(f"{output_dir}/{file}")
    else:
        logger.warning(f"Directory {output_dir} does not exist. Creating it.")
        Path(output_dir).mkdir(parents=True)

    algorithms = pd.unique(data["Algorithm"])
    problems = pd.unique(data["Problem"])
    indicators = pd.unique(data["IndicatorName"])

    control_algorithm = algorithms[-1]

    # Create data frame to store the Kolmogorov Smirnov test results
    test_data = pd.DataFrame(columns=["Indicator", "Algorithm", "Problem", "PValue", "TestResult"])

    for indicator in indicators:
        for algorithm in algorithms:
            for problem in problems:
                algorithm_data = data[
                    (data["Problem"] == problem)
                    & (data["Algorithm"] == algorithm)
                    & (data["IndicatorName"] == indicator)
                ]
                ref_data = data[
                    (data["Problem"] == problem)
                    & (data["Algorithm"] == control_algorithm)
                    & (data["IndicatorName"] == indicator)
                ]
                stat, p_value = ks_2samp(
                    algorithm_data["IndicatorValue"], ref_data["IndicatorValue"]
                )

                test_result = stat

                new_row = {
                    "Indicator": indicator,
                    "Algorithm": algorithm,
                    "Problem": problem,
                    "PValue": p_value,
                    "TestResult": test_result,
                }
                test_data = test_data._append(new_row, ignore_index=True)

    # Generate LaTeX tables
    caption = (
        "Kolmogorov-Smirnov Test of the {} quality indicator. "
        "The algorithm in the last column is the reference "
        + "algorithm and each cell contain the p-value obtained when applying the test with the reference "
        "algorithm. Cells with gray background highlight p-values less than 0.05 (i.e., the null hypothesis"
        " -- the two distributions are identical -- is rejected)."
    )
    for indicator_name in indicators:
        with open(
            os.path.join(output_dir, f"KolmogorovSmirnov-{indicator_name}.tex"), "w"
        ) as latex:
            latex.write(
                __kolmogorov_smirnov_to_latex(
                    indicator_name,
                    test_data,
                    caption=caption.format(indicator_name),
                    label=f"table:{indicator_name}",
                )
            )