-
Notifications
You must be signed in to change notification settings - Fork 0
/
sweep.py
executable file
·142 lines (130 loc) · 6.96 KB
/
sweep.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import os
import argparse
parser = argparse.ArgumentParser(description="sweep parser")
parser.add_argument("--lr", type=float, default=0.00088)
parser.add_argument("--batch-size", type=int, default=16)
parser.add_argument("--rehearsal-ratio", type=int, default=30)
parser.add_argument("--num-candidates", type=int, default=8)
parser.add_argument("--num-representatives", type=int, default=24)
parser.add_argument("--augmentations-offset", type=int, default=16)
parser.add_argument("--soft-augmentations-offset", type=int, default=25)
parser.add_argument("--epochs", type=int, default=1)
parser.add_argument("--weight-decay", type=float, default=0)
parser.add_argument("--lr-schedule", type=str, default="exp_range_cyclic_lr")
parser.add_argument("--step-cycle-size", type=int, default=2944)
parser.add_argument("--alpha", type=float, default=0.8)
parser.add_argument("--beta", type=float, default=1.0)
args = parser.parse_args()
host = ""
sweep_conf = ""
er_command = (
f"horovodrun -np 8 -H {host} python main.py "
f"--backbone resnet50 "
f"--model Er "
f"--dataset imagenet_blurred "
f"--dataset-dir /my-spack/datasets "
f"--tasksets-config \"{{'scenario': 'class', 'initial_increment': 250, 'increment': 250}}\" "
f"--buffer-config \"{{'implementation': 'standard', 'num_samples_per_representative': 1, 'num_candidates': {args.num_candidates}, 'num_representatives': {args.num_representatives}, 'rehearsal_ratio': 30, 'augmentations_offset': {args.augmentations_offset}, 'provider': 'na+sm', 'discover_endpoints': True, 'cuda_rdma': False}}\" "
f"--batch-size 56 "
f"--lr 0.0125 "
f"--epochs 30 "
f"--warmup-epochs 5 "
f"--use-amp "
f"--use-dali "
f"--load-checkpoint /root/distributed-continual-learning/checkpoint_initial_resnet50_imagenet_1.pth.tar "
)
der_command = (
f"horovodrun -np 8 -H {host} python main.py "
f"--backbone resnet50 "
f"--model Der "
f"--model-config \"{{'alpha': {args.alpha}}}\" "
f"--dataset imagenet_blurred "
f"--dataset-dir /my-spack/datasets "
f"--tasksets-config \"{{'scenario': 'class', 'initial_increment': 250, 'increment': 250}}\" "
f"--buffer-config \"{{'implementation': 'standard', 'num_samples_per_representative': 1, 'num_samples_per_activation': 1, 'activation_shape': [1000], 'num_candidates': {args.num_candidates}, 'num_representatives': {args.num_representatives}, 'rehearsal_ratio': 30, 'augmentations_offset': {args.augmentations_offset}, 'provider': 'na+sm', 'discover_endpoints': True, 'cuda_rdma': False}}\" "
f"--batch-size 56 "
f"--lr 0.0125 "
f"--epochs 30 "
f"--warmup-epochs 5 "
f"--use-amp "
f"--use-dali "
f"--load-checkpoint /root/distributed-continual-learning/checkpoint_initial_resnet50_imagenet_1.pth.tar "
)
baseline_scratch_ptycho_command = (
f"horovodrun -np 8 -H {host} python main.py "
f"--backbone ptychonn "
f"--backbone-config \"{{'lr_schedule': '{args.lr_schedule}', 'step_cycle_size': {args.step_cycle_size}, 'weight_decay': {args.weight_decay}}}\" "
f"--model Vanilla "
f"--dataset ptycho "
f"--dataset-dir /my-spack/datasets "
f"--tasksets-config \"{{'scenario': 'reconstruction', 'num_tasks': 156, 'concatenate_tasksets': True}}\" "
f"--batch-size {args.batch_size} "
f"--eval-batch-size 5 "
f"--lr {args.lr} "
f"--epochs {args.epochs} "
f"--use-amp "
f"--use-dali "
f"--load-checkpoint /root/distributed-continual-learning/checkpoint_initial_ptychonn_1.pth.tar "
)
er_ptycho_command = (
f"horovodrun -np 8 -H {host} python main.py "
f"--backbone ptychonn "
f"--backbone-config \"{{'lr_schedule': '{args.lr_schedule}', 'step_cycle_size': {args.step_cycle_size}, 'weight_decay': {args.weight_decay}}}\" "
f"--model Er "
f"--dataset ptycho "
f"--dataset-dir /my-spack/datasets "
f"--tasksets-config \"{{'scenario': 'reconstruction', 'num_tasks': 156}}\" "
f"--buffer-config \"{{'implementation': 'standard', 'num_samples_per_representative': 3, 'num_candidates': {args.num_candidates}, 'num_representatives': {args.num_representatives}, 'rehearsal_ratio': {args.rehearsal_ratio}, 'augmentations_offset': {args.augmentations_offset}, 'provider': 'na+sm', 'discover_endpoints': True, 'cuda_rdma': False}}\" "
f"--batch-size {args.batch_size} "
f"--eval-batch-size 5 "
f"--lr {args.lr} "
f"--epochs {args.epochs} "
f"--use-amp "
f"--use-dali "
f"--load-checkpoint /root/distributed-continual-learning/checkpoint_initial_ptychonn_1.pth.tar "
)
derpp_ptycho_command = (
f"horovodrun -np 8 -H {host} python main.py "
f"--backbone ptychonn "
f"--backbone-config \"{{'lr_schedule': '{args.lr_schedule}', 'step_cycle_size': {args.step_cycle_size}, 'weight_decay': {args.weight_decay}}}\" "
f"--model Derpp "
f"--model-config \"{{'alpha': {args.alpha}, 'beta': {args.beta}}}\" "
f"--dataset ptycho "
f"--dataset-dir /my-spack/datasets "
f"--tasksets-config \"{{'scenario': 'reconstruction', 'num_tasks': 156}}\" "
f"--buffer-config \"{{'implementation': 'standard', 'num_samples_per_representative': 3, 'num_candidates': {args.num_candidates}, 'num_representatives': {args.num_representatives}, 'rehearsal_ratio': {args.rehearsal_ratio}, 'augmentations_offset': {args.augmentations_offset}, 'provider': 'na+sm', 'discover_endpoints': True, 'cuda_rdma': False}}\" "
f"--batch-size {args.batch_size} "
f"--eval-batch-size 5 "
f"--lr {args.lr} "
f"--epochs {args.epochs} "
f"--use-amp "
f"--use-dali "
f"--load-checkpoint /root/distributed-continual-learning/checkpoint_initial_ptychonn_1.pth.tar "
)
beta_derpp_ptycho_command = (
f"horovodrun -np 8 -H {host} python main.py "
f"--backbone ptychonn "
f"--backbone-config \"{{'lr_schedule': '{args.lr_schedule}', 'step_cycle_size': {args.step_cycle_size}, 'weight_decay': {args.weight_decay}}}\" "
f"--model Derpp "
f"--model-config \"{{'alpha': {args.alpha}, 'beta': {args.beta}}}\" "
f"--dataset ptycho "
f"--dataset-dir /my-spack/datasets "
f"--tasksets-config \"{{'scenario': 'reconstruction', 'num_tasks': 156}}\" "
f"--buffer-config \"{{'implementation': 'standard', 'num_samples_per_representative': 3, 'num_candidates': {args.num_candidates}, 'num_representatives': {args.num_representatives}, 'rehearsal_ratio': {args.rehearsal_ratio}, 'augmentations_offset': {args.augmentations_offset}, 'soft_augmentations_offset': {args.soft_augmentations_offset}, 'provider': 'na+sm', 'discover_endpoints': True, 'cuda_rdma': False}}\" "
f"--batch-size {args.batch_size} "
f"--eval-batch-size 5 "
f"--lr {args.lr} "
f"--epochs 1 "
f"--use-amp "
f"--use-dali "
f"--load-checkpoint /root/distributed-continual-learning/checkpoint_initial_ptychonn_1.pth.tar "
)
command = {
"er": er_command,
"der": der_command,
"baseline_scratch_ptycho": baseline_scratch_ptycho_command,
"er_ptycho": er_ptycho_command,
"derpp_ptycho": derpp_ptycho_command,
"beta_derpp_ptycho": beta_derpp_ptycho_command,
}[sweep_conf]
os.system(command)