diff --git a/docs/doc_benchmark.md b/docs/doc_benchmark.md index 759459c32..cbc3d3e48 100644 --- a/docs/doc_benchmark.md +++ b/docs/doc_benchmark.md @@ -126,8 +126,8 @@ Furthermore, the user can declare: - `startseed`: creates reproducible results (mandatory) - `endseed`: creates reproducible results (mandatory) - `mode`: set to `grid` to apply grid search for hyperparameter sampling (optional, for details see next section) -- `Shared params`: an optional list including the shared hyperparameters with respective sampling distribution -(mandatory if Shared params should be used) and in case of random sampling `num_shared_param_samples` +- `commonHyperpars2sample`: an optional list including the shared hyperparameters with respective sampling distribution +(mandatory if shared parameters should be used) and in case of random sampling `num_shared_param_samples` (number of samples for the shared hyperparameters, mandatory for random sampling) Depending on which hyperparameter sampling technique is used (see section below), the user must also @@ -140,7 +140,7 @@ respect/declare the following: - `step`: "step-size" (float) between samples. Only points being a multiple of the step-size apart can be sampled. `0` means that each real number can be sampled. - `num_shared_param_samples`: number of samples for the shared hyperparameters. Must be defined - inside the Shared params section (if this section is used) + inside the commonHyperpars2sample section (if this section is used) - grid search hyperparameter sampling (`mode`:`grid`): - `num`: number of hyperparameters to be sampled (int) must be specified for each hyperparameter (mandatory) diff --git a/docs/doc_benchmark_yaml.md b/docs/doc_benchmark_yaml.md index 1e7dce366..5f7d7e04f 100644 --- a/docs/doc_benchmark_yaml.md +++ b/docs/doc_benchmark_yaml.md @@ -85,7 +85,7 @@ domainlab_args: ################################################################################### # Hyperparameters which appear in multiple tasks can e shared among these tasks. # Hence for each task the same random sampled hyperparameters are used -Shared params: +commonHyperpars2sample: #### For random sampling ##### # number of shared samples to be created. # The sampling procedure creates a set of randomly sampled shared samples, @@ -155,7 +155,7 @@ Task_Dann: There are two possible ways of choosing your hyperparameters for the benchmark in domainlab, rand sampling and grid search. The decision about which method to use was already done in the previous section be either setting `num_param_samples` (for random sampling) or `mode: grid` (for gridsearch). -For filling in the sampling description for the into the `Shared params` and the `hyperparameter` section you have the following options: +For filling in the sampling description for the into the `commonHyperpars2sample` and the `hyperparameter` section you have the following options: ### uniform and loguniform distribution 1. uniform samples in the interval [min, max] @@ -252,7 +252,7 @@ Task_Name: aname: ... ... - # specify sections from the Shared params section + # specify sections from the commonHyperpars2sample section shared: - ... # specify task specific hyperparameter sampling @@ -283,7 +283,7 @@ Task_Name: aname: ... ... - # specify sections from the Shared params section + # specify sections from the commonHyperpars2sample section shared: - ... # add the constraints as a standalone section to the task diff --git a/domainlab/utils/hyperparameter_gridsearch.py b/domainlab/utils/hyperparameter_gridsearch.py index 522196046..fa32e9fcf 100644 --- a/domainlab/utils/hyperparameter_gridsearch.py +++ b/domainlab/utils/hyperparameter_gridsearch.py @@ -332,8 +332,8 @@ def sample_gridsearch(config: dict, samples = pd.DataFrame(columns=['task', 'algo', 'params']) shared_samples_full = pd.DataFrame(columns=['task', 'algo', 'params']) - if 'Shared params' in config.keys(): - shared_val = {'aname': 'all', 'hyperparameters': config['Shared params']} + if 'commonHyperpars2sample' in config.keys(): + shared_val = {'aname': 'all', 'hyperparameters': config['commonHyperpars2sample']} # fill up the dataframe shared samples shared_samples_full = grid_task(shared_samples_full, 'all', shared_val, None) else: diff --git a/domainlab/utils/hyperparameter_sampling.py b/domainlab/utils/hyperparameter_sampling.py index d4964781b..f191a0a66 100644 --- a/domainlab/utils/hyperparameter_sampling.py +++ b/domainlab/utils/hyperparameter_sampling.py @@ -411,10 +411,10 @@ def sample_hyperparameters(config: dict, num_samples = config['num_param_samples'] samples = pd.DataFrame(columns=['task', 'algo', 'params']) - shared_config_full = config['Shared params'] + shared_config_full = config['commonHyperpars2sample'] shared_samples_full = pd.DataFrame(columns=['task', 'algo', 'params']) - if 'Shared params' in config.keys(): - shared_val = {'aname': 'all', 'hyperparameters': config['Shared params']} + if 'commonHyperpars2sample' in config.keys(): + shared_val = {'aname': 'all', 'hyperparameters': config['commonHyperpars2sample']} # fill up the dataframe shared samples shared_samples_full = sample_task(shared_config_full['num_shared_param_samples'], 'all', (shared_val, shared_samples_full), (None, None)) diff --git a/examples/benchmark/benchmark_mnist_shared_hyper_grid.yaml b/examples/benchmark/benchmark_mnist_shared_hyper_grid.yaml index 4dbdd313d..9cb26e09b 100644 --- a/examples/benchmark/benchmark_mnist_shared_hyper_grid.yaml +++ b/examples/benchmark/benchmark_mnist_shared_hyper_grid.yaml @@ -10,7 +10,7 @@ startseed: 0 endseed: 2 # currently included -Shared params: +commonHyperpars2sample: zy_dim: min: 32 max: 96 diff --git a/examples/benchmark/benchmark_pacs_resnet.yaml b/examples/benchmark/benchmark_pacs_resnet.yaml index 55d1fc381..7cdf4a43c 100644 --- a/examples/benchmark/benchmark_pacs_resnet.yaml +++ b/examples/benchmark/benchmark_pacs_resnet.yaml @@ -30,7 +30,7 @@ domainlab_args: npath_encoder_sandwich_layer_img2h4zd: examples/nets/resnet50domainbed.py san_check: True -Shared params: +commonHyperpars2sample: num_shared_param_samples: 8 gamma_y: min: 1e4 diff --git a/examples/benchmark/benchmark_pacs_resnet_on_mnist.yaml b/examples/benchmark/benchmark_pacs_resnet_on_mnist.yaml index b59f9a6c3..362008634 100644 --- a/examples/benchmark/benchmark_pacs_resnet_on_mnist.yaml +++ b/examples/benchmark/benchmark_pacs_resnet_on_mnist.yaml @@ -32,7 +32,7 @@ domainlab_args: nname_encoder_sandwich_layer_img2h4zd: conv_bn_pool_2 san_check: True -Shared params: +commonHyperpars2sample: num_shared_param_samples: 8 lr: diff --git a/examples/yaml/demo_hyperparameter_gridsearch.yml b/examples/yaml/demo_hyperparameter_gridsearch.yml index b98c283e2..2cc071b6f 100644 --- a/examples/yaml/demo_hyperparameter_gridsearch.yml +++ b/examples/yaml/demo_hyperparameter_gridsearch.yml @@ -2,7 +2,7 @@ output_dir: "zoutput/benchmarks/hyperparameter_grid_test" mode: grid -Shared params: +commonHyperpars2sample: p1_shared: distribution: uniform min: 1 diff --git a/examples/yaml/demo_hyperparameter_sampling.yml b/examples/yaml/demo_hyperparameter_sampling.yml index 7573539de..43e75456e 100644 --- a/examples/yaml/demo_hyperparameter_sampling.yml +++ b/examples/yaml/demo_hyperparameter_sampling.yml @@ -3,7 +3,7 @@ output_dir: "zoutput/benchmarks/hyperparameter_test" num_param_samples: 30 -Shared params: +commonHyperpars2sample: num_shared_param_samples: 5 p1_shared: distribution: uniform