Tuning Analysis

This page provides an analysis of the hyperparameter sweeps to get a handle on how different tuning methods are performing. We use these results from early data sets to select the most efficient strategies for other data sets.

All results are validation error.

Trial Performance

These charts display the maximum performance so far as the search progresses through its trials. The vertical lines are at 60 trials.

RBP

NDCG

Note

Intelligent search methods (HyperOpt and Optuna) are searching for RBP, so the NDCG performance may not be fully reflective.

Loss

Our goal on the smaller MovieLens data sets, using longer searches, is to determine where to stop searching, and which methods are more efficient at searching the space. To better assess this, let’s look at the loss relative to each method’s best performance if we stop at each trial point.

Exploring Loss Loss

This is an interactive display so we can directly examine the results of stopping the search at different trial counts.

viewof max_trial = Inputs.range(
  [20, ntrials],
  {value: ntrials, step: 5, label: "Maximum trials:"}
)

filt_runs = transpose(runs).filter((run) => run.TrialNum <= max_trial)
best_list = transpose(best_rbps)
rbp_lookup = Object.fromEntries(best_list.map((m) => [m.model, m]))

function makeRunTable(cutoff) {
    let tbl = [];
    for (let mrec of best_list) {
        let row = {model: mrec.model};
        for (let search in mrec) {
            if (search == 'model') continue;

            let runs = filt_runs.filter((r) => r.model == mrec.model && r.search == search && r.TrialNum <= cutoff);
            let run = runs[runs.length - 1]
            if (run) {
                row[search] = `${run.MaxRBP.toFixed(3)} / ${mrec[search].toFixed(3)}`;
            }
        }
        tbl.push(row);
    }
    return tbl;
}

search_tbl = makeRunTable(max_trial);
Inputs.table(search_tbl)