diff --git a/downstream/Clone-detection-CodeNet/run.sh b/downstream/Clone-detection-CodeNet/run.sh index 5bf3f0b..9ff115a 100755 --- a/downstream/Clone-detection-CodeNet/run.sh +++ b/downstream/Clone-detection-CodeNet/run.sh @@ -21,7 +21,7 @@ python ./code/run.py \ --epoch 2 \ --block_size 400 \ --train_batch_size 8 \ - --eval_batch_size 16 \ + --eval_batch_size 64 \ --learning_rate 2e-5 \ --max_grad_norm 1.0 \ --evaluate_during_training \ diff --git a/downstream/Clone-detection-CodeNet/run_aug_test.sh b/downstream/Clone-detection-CodeNet/run_aug_test.sh index ff12c8b..515f73d 100755 --- a/downstream/Clone-detection-CodeNet/run_aug_test.sh +++ b/downstream/Clone-detection-CodeNet/run_aug_test.sh @@ -20,7 +20,7 @@ python ./code/run.py \ --test_predictions_file=aug_predictions.jsonl \ --epoch 2 \ --block_size 400 \ - --eval_batch_size 16 \ + --eval_batch_size 64 \ --learning_rate 2e-5 \ --max_grad_norm 1.0 \ --seed 123456 2>&1| tee $output_dir/aug_train.log diff --git a/downstream/Clone-detection-POJ104/run.sh b/downstream/Clone-detection-POJ104/run.sh index 88d7939..f8a5347 100755 --- a/downstream/Clone-detection-POJ104/run.sh +++ b/downstream/Clone-detection-POJ104/run.sh @@ -17,7 +17,7 @@ python ./code/run.py \ --epoch 2 \ --block_size 400 \ --train_batch_size 8 \ - --eval_batch_size 16 \ + --eval_batch_size 64 \ --learning_rate 2e-5 \ --max_grad_norm 1.0 \ --evaluate_during_training \ diff --git a/downstream/Clone-detection-POJ104/run_aug_test.sh b/downstream/Clone-detection-POJ104/run_aug_test.sh index dbbe33c..db4aed4 100755 --- a/downstream/Clone-detection-POJ104/run_aug_test.sh +++ b/downstream/Clone-detection-POJ104/run_aug_test.sh @@ -19,7 +19,7 @@ python ./code/run.py \ --test_predictions_file=aug_predictions.jsonl \ --epoch 2 \ --block_size 400 \ - --eval_batch_size 16 \ + --eval_batch_size 64 \ --learning_rate 2e-5 \ --max_grad_norm 1.0 \ --seed 123456 2>&1| tee $output_dir/aug_train.log diff --git a/downstream/Code-classification-CodeNet/run_aug_test.sh b/downstream/Code-classification-CodeNet/run_aug_test.sh index 1c656d9..cccafbb 100755 --- a/downstream/Code-classification-CodeNet/run_aug_test.sh +++ b/downstream/Code-classification-CodeNet/run_aug_test.sh @@ -19,7 +19,7 @@ python ./code/run.py \ --num_train_epochs 5 \ --block_size 256 \ --train_batch_size 8 \ - --eval_batch_size 16 \ + --eval_batch_size 64 \ --learning_rate 2e-5 \ --max_grad_norm 1.0 \ --seed 123456 2>&1 | tee $output_dir/aug_test.log diff --git a/downstream/Code-classification-POJ104/run_aug_test.sh b/downstream/Code-classification-POJ104/run_aug_test.sh index 1c656d9..cccafbb 100755 --- a/downstream/Code-classification-POJ104/run_aug_test.sh +++ b/downstream/Code-classification-POJ104/run_aug_test.sh @@ -19,7 +19,7 @@ python ./code/run.py \ --num_train_epochs 5 \ --block_size 256 \ --train_batch_size 8 \ - --eval_batch_size 16 \ + --eval_batch_size 64 \ --learning_rate 2e-5 \ --max_grad_norm 1.0 \ --seed 123456 2>&1 | tee $output_dir/aug_test.log diff --git a/experiments/grouped/codebert-base.yaml b/experiments/grouped/codebert-base.yaml deleted file mode 100644 index 34c8c87..0000000 --- a/experiments/grouped/codebert-base.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Grouped multi-key contrastive pre-training -# Usage: python -m modeling.pretrain --config experiments/grouped_example.yaml - -dataset_path: "data/aug_csn.jsonl" -model_name: "microsoft/codebert-base" -tokenizer_name: "microsoft/codebert-base" - -batch_size: 2 -num_epochs: 3 -gradient_accumulation_steps: 8 -learning_rate: 2.0e-5 - -seed: 42 - -# naming: model name + contra mode + aug (aug only / all) + sample rate -run_name: "codebert-grouped-aug-0.2" - -alpha: 1.0 -temperature: 0.1 -max_seq_length: 512 -sample_rate: 0.2 - -contra_mode: "grouped" -max_num_augs: 6 diff --git a/experiments/grouped/codebert.yaml b/experiments/grouped/codebert.yaml new file mode 100644 index 0000000..addef07 --- /dev/null +++ b/experiments/grouped/codebert.yaml @@ -0,0 +1,22 @@ +# InvPT pre-training: Grouped — CodeBERT +# Effective batch size: 16 * 16 = 256 +# Usage: python modeling/cli.py run experiments/grouped/codebert.yaml + +dataset_path: "data/aug_csn.jsonl" +model_name: "microsoft/codebert-base" + +batch_size: 16 +num_epochs: 3 +gradient_accumulation_steps: 16 +learning_rate: 2.0e-5 + +seed: 0 +run_name: "InvCodeBERT-grouped" + +alpha: 1.0 +temperature: 0.1 +max_seq_length: 512 +sample_rate: 0.2 + +contra_mode: "grouped" +max_num_augs: 6 diff --git a/experiments/grouped/contrabert_c.yaml b/experiments/grouped/contrabert_c.yaml new file mode 100644 index 0000000..1f8e87c --- /dev/null +++ b/experiments/grouped/contrabert_c.yaml @@ -0,0 +1,23 @@ +# InvPT pre-training: Grouped — ContraBERT_C +# Effective batch size: 16 * 16 = 256 +# Usage: python modeling/cli.py run experiments/grouped/contrabert_c.yaml + +dataset_path: "data/aug_csn.jsonl" +model_name: "./saved_models/ContraBERT_C" +tokenizer_name: "microsoft/codebert-base" + +batch_size: 16 +num_epochs: 3 +gradient_accumulation_steps: 16 +learning_rate: 2.0e-5 + +seed: 0 +run_name: "InvContraBERT_C-grouped" + +alpha: 1.0 +temperature: 0.1 +max_seq_length: 512 +sample_rate: 0.2 + +contra_mode: "grouped" +max_num_augs: 6 diff --git a/experiments/grouped/contrabert_g.yaml b/experiments/grouped/contrabert_g.yaml new file mode 100644 index 0000000..0b8db32 --- /dev/null +++ b/experiments/grouped/contrabert_g.yaml @@ -0,0 +1,23 @@ +# InvPT pre-training: Grouped — ContraBERT_G +# Effective batch size: 16 * 16 = 256 +# Usage: python modeling/cli.py run experiments/grouped/contrabert_g.yaml + +dataset_path: "data/aug_csn.jsonl" +model_name: "./saved_models/ContraBERT_G" +tokenizer_name: "microsoft/graphcodebert-base" + +batch_size: 16 +num_epochs: 3 +gradient_accumulation_steps: 16 +learning_rate: 2.0e-5 + +seed: 0 +run_name: "InvContraBERT_G-grouped" + +alpha: 1.0 +temperature: 0.1 +max_seq_length: 512 +sample_rate: 0.2 + +contra_mode: "grouped" +max_num_augs: 6 diff --git a/experiments/grouped/graphcodebert.yaml b/experiments/grouped/graphcodebert.yaml new file mode 100644 index 0000000..0e2d204 --- /dev/null +++ b/experiments/grouped/graphcodebert.yaml @@ -0,0 +1,22 @@ +# InvPT pre-training: Grouped — GraphCodeBERT +# Effective batch size: 16 * 16 = 256 +# Usage: python modeling/cli.py run experiments/grouped/graphcodebert.yaml + +dataset_path: "data/aug_csn.jsonl" +model_name: "microsoft/graphcodebert-base" + +batch_size: 16 +num_epochs: 3 +gradient_accumulation_steps: 16 +learning_rate: 2.0e-5 + +seed: 0 +run_name: "InvGraphCodeBERT-grouped" + +alpha: 1.0 +temperature: 0.1 +max_seq_length: 512 +sample_rate: 0.2 + +contra_mode: "grouped" +max_num_augs: 6 diff --git a/experiments/supcon/codebert.yaml b/experiments/supcon/codebert.yaml new file mode 100644 index 0000000..477d1f4 --- /dev/null +++ b/experiments/supcon/codebert.yaml @@ -0,0 +1,21 @@ +# InvPT pre-training: SupCon — CodeBERT +# Effective batch size: 64 * 4 = 256 +# Usage: python modeling/cli.py run experiments/supcon/codebert.yaml + +dataset_path: "data/aug_csn.jsonl" +model_name: "microsoft/codebert-base" + +batch_size: 64 +num_epochs: 3 +gradient_accumulation_steps: 4 +learning_rate: 2.0e-5 + +seed: 0 +run_name: "InvCodeBERT-supcon" + +alpha: 1.0 +temperature: 0.1 +max_seq_length: 512 +sample_rate: 0.2 + +contra_mode: "supcon" diff --git a/experiments/supcon/contrabert_c.yaml b/experiments/supcon/contrabert_c.yaml new file mode 100644 index 0000000..28704bb --- /dev/null +++ b/experiments/supcon/contrabert_c.yaml @@ -0,0 +1,22 @@ +# InvPT pre-training: SupCon — ContraBERT_C +# Effective batch size: 64 * 4 = 256 +# Usage: python modeling/cli.py run experiments/supcon/contrabert_c.yaml + +dataset_path: "data/aug_csn.jsonl" +model_name: "./saved_models/ContraBERT_C" +tokenizer_name: "microsoft/codebert-base" + +batch_size: 64 +num_epochs: 3 +gradient_accumulation_steps: 4 +learning_rate: 2.0e-5 + +seed: 0 +run_name: "InvContraBERT_C-supcon" + +alpha: 1.0 +temperature: 0.1 +max_seq_length: 512 +sample_rate: 0.2 + +contra_mode: "supcon" diff --git a/experiments/base.yaml b/experiments/supcon/contrabert_g.yaml similarity index 58% rename from experiments/base.yaml rename to experiments/supcon/contrabert_g.yaml index a3f464e..4f6b973 100644 --- a/experiments/base.yaml +++ b/experiments/supcon/contrabert_g.yaml @@ -1,5 +1,6 @@ -# InvPT pre-training experiment configuration -# Usage: python -m modeling.pretrain --config experiments/base.yaml +# InvPT pre-training: SupCon — ContraBERT_G +# Effective batch size: 64 * 4 = 256 +# Usage: python modeling/cli.py run experiments/supcon/contrabert_g.yaml dataset_path: "data/aug_csn.jsonl" model_name: "./saved_models/ContraBERT_G" @@ -11,7 +12,7 @@ gradient_accumulation_steps: 4 learning_rate: 2.0e-5 seed: 0 -run_name: "InvContraBERT_G-aug02-supcon" +run_name: "InvContraBERT_G-supcon" alpha: 1.0 temperature: 0.1 @@ -19,4 +20,3 @@ max_seq_length: 512 sample_rate: 0.2 contra_mode: "supcon" -# max_num_augs: 6 # only used with contra_mode: "grouped" diff --git a/experiments/supcon/graphcodebert.yaml b/experiments/supcon/graphcodebert.yaml new file mode 100644 index 0000000..8444138 --- /dev/null +++ b/experiments/supcon/graphcodebert.yaml @@ -0,0 +1,21 @@ +# InvPT pre-training: SupCon — GraphCodeBERT +# Effective batch size: 64 * 4 = 256 +# Usage: python modeling/cli.py run experiments/supcon/graphcodebert.yaml + +dataset_path: "data/aug_csn.jsonl" +model_name: "microsoft/graphcodebert-base" + +batch_size: 64 +num_epochs: 3 +gradient_accumulation_steps: 4 +learning_rate: 2.0e-5 + +seed: 0 +run_name: "InvGraphCodeBERT-supcon" + +alpha: 1.0 +temperature: 0.1 +max_seq_length: 512 +sample_rate: 0.2 + +contra_mode: "supcon"