diff --git a/AGENTS.md b/AGENTS.md index 58055e4..7afb14c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,8 @@ InvPT/ │ └── visualize.py ├── experiments/ # YAML experiment configurations │ ├── base.yaml # Base supcon config (matches original run_pretrain.sh) -│ └── grouped_example.yaml # Grouped contrastive mode example +│ ├── grouped_example.yaml # Grouped contrastive mode example +│ └── modernbert_base.yaml # ModernBERT-base config (mean pooling) ├── saved_models/ # Pre-trained model checkpoints ├── run_pretrain.sh # Pre-training launch script ├── clang.sh # LLVM 14 installation script @@ -64,6 +65,8 @@ InvPT/ - **PL-only**: Unlike prior work (CodeBERT, ContraBERT), InvPT removes natural language docstrings during pre-training. - **No MoCo**: Uses a single shared encoder for original and transformed code, unlike ContraBERT which uses momentum contrast. - **Contrastive modes**: `info_nce` (diagonal positives), `supcon` (multi-positive by function_id mask), `grouped` (grouped multi-key contrast with explicit aug grouping via `--max_num_augs`). +- **Model types**: `roberta` (RoBERTa/CodeBERT/ContraBERT, default) and `modernbert` (ModernBERT with RoPE, Flash Attention, 8K context). Configured via `model_type` in YAML configs. +- **Pooling strategies**: `cls` (CLS token, default for RoBERTa) and `mean` (mean pooling over non-padding tokens, recommended for ModernBERT). ## Transformation Operators @@ -84,7 +87,7 @@ source .envrc # Load environment variables ./clang.sh # Install LLVM 14 (for C/C++ transforms) ``` -Requires: Python 3.11+, JDK 11+ (for Java transforms), LLVM 14 (for C/C++ transforms). +Requires: Python 3.11+, JDK 11+ (for Java transforms), LLVM 14 (for C/C++ transforms), transformers >= 4.48 (for ModernBERT support). ## Running Tests diff --git a/doc/modernbert.md b/doc/modernbert.md new file mode 100644 index 0000000..f34cd6f --- /dev/null +++ b/doc/modernbert.md @@ -0,0 +1,268 @@ +# ModernBERT Support + +## Overview + +InvPT supports [ModernBERT](https://github.com/AnswerDotAI/ModernBERT) as an alternative +backbone to RoBERTa/CodeBERT/ContraBERT. ModernBERT is a modernized BERT architecture +from AnswerDotAI that incorporates recent advances in transformer design while retaining +the encoder-only, MLM-based pre-training paradigm that InvPT builds on. + +Two variants are available on HuggingFace: + +| Variant | Layers | Hidden | Params | HF Identifier | +| ------- | ------ | ------ | ------ | ------------------------------ | +| Base | 22 | 768 | 149M | `answerdotai/ModernBERT-base` | +| Large | 28 | 1024 | 395M | `answerdotai/ModernBERT-large` | + +## Why ModernBERT for InvPT + +### Trained on Code + +ModernBERT was pre-trained on 2 trillion tokens of English text **and code**. +RoBERTa (and by extension CodeBERT/ContraBERT) was trained on ~160GB of English text only. +This means ModernBERT already has a code-aware representation before InvPT's continued +pre-training, giving it a stronger starting point for learning invariant code representations. + +### Longer Context (8,192 tokens) + +RoBERTa's absolute position embeddings cap out at 512 tokens. ModernBERT uses Rotary +Position Embeddings (RoPE) with a maximum of 8,192 tokens. This allows processing longer +functions without truncation — particularly useful for languages like Java and C++ where +functions routinely exceed 512 tokens. + +### Architectural Improvements + +| Feature | RoBERTa | ModernBERT | +| ------------------- | --------------------------- | ------------------------------------------------------------------- | +| Position embeddings | Absolute (learned, 512 max) | RoPE (8,192 max) | +| Attention pattern | Full attention all layers | Local-global alternating (sliding window 128 + full every 3 layers) | +| Attention backend | Standard SDPA | Flash Attention 2 + unpadding | +| FFN activation | GELU | GeGLU (gated) | +| Normalization | Post-LayerNorm | Pre-Norm (no bias) | +| Weight tying | No | Yes (embeddings tied to decoder) | + +The local-global alternating attention is especially relevant: it reduces the quadratic +cost of full attention on long sequences while preserving global information flow through +periodic full-attention layers (every 3rd layer). + +Flash Attention and unpadding (skipping compute on padding tokens) provide significant +speedups, particularly for variable-length code batches where padding waste is high. + +## Configuration + +### Experiment Config + +Create a YAML config in `experiments/` (or use the provided `modernbert_base.yaml`): + +```yaml +model_name: "answerdotai/ModernBERT-base" +model_type: "modernbert" +pooling: "mean" +max_seq_length: 512 +run_name: "InvPT-ModernBERT-base" +``` + +Two new fields control the behavior: + +- **`model_type`**: Either `"roberta"` (default) or `"modernbert"`. Determines how + the wrapper accesses the encoder and LM head internals. +- **`pooling`**: Either `"cls"` (default, CLS token) or `"mean"` (mean over non-padding + tokens). Mean pooling is recommended for ModernBERT (see below). + +All other config fields (`alpha`, `temperature`, `contra_mode`, etc.) work identically. + +### CLI Usage + +```bash +# From YAML config (recommended) +python modeling/cli.py run experiments/modernbert_base.yaml + +# Direct CLI options +python modeling/cli.py pretrain \ + --model-name answerdotai/ModernBERT-base \ + --model-type modernbert \ + --pooling mean \ + --max-seq-length 512 +``` + +## Training Pipeline + +The training pipeline is **architecture-agnostic** — ModernBERT uses the exact same +loss computation, contrastive learning, and curriculum as RoBERTa/CodeBERT/ContraBERT. +The only ModernBERT-specific logic lives in `SplitHeadWrapper` (encoder/LM-head dispatch) +and the `pooling` config field. + +### Loss Function + +The total loss is identical for both model types: + +``` +L = L_MLM(code) + L_MLM(aug) + alpha * L_contrastive(code, aug) +``` + +- **MLM loss**: 15% random token masking applied independently to both the original code + and its augmentation. Computed per-chunk via `SplitHeadWrapper` to avoid materializing + the full `[2B, seq_len, vocab_size]` logit tensor. +- **Contrastive loss**: Computed on pooled embeddings (CLS or mean) from the shared + encoder's last hidden states. Controlled by `alpha` (default 1.0) and `temperature` + (default 0.07). + +### Contrastive Modes + +All three contrastive modes work with ModernBERT: + +| Mode | Config value | Description | +| ------- | ------------ | --------------------------------------------------------------------------------------------- | +| InfoNCE | `info_nce` | Diagonal positives — each code paired with its single augmentation | +| SupCon | `supcon` | Multi-positive by `function_id` — all augmentations of the same function are mutual positives | +| Grouped | `grouped` | Explicit grouped multi-key contrast with up to `max_num_augs` augmentations per anchor | + +Set via the `contra_mode` field in the YAML config. + +### Self-Contrast + +Self-contrast (`self_contrast: true`, the default) provides the "easy" curriculum signal. +When a dataset row has no successful transformation (e.g., the code had no variables to +rename), the original code is reused as its own augmentation. The contrastive signal then +comes from **different MLM masks** applied to identical code — the encoder must learn that +the same code under different masks maps to the same representation. + +This is independent of model type and works identically for ModernBERT. When +`self_contrast` is disabled, rows without transformations are dropped from the dataset. + +### What Differs + +| Aspect | RoBERTa | ModernBERT | +| ----------------------- | ----------------------------------------- | ----------------------------- | +| Loss function | `L_MLM + alpha * L_contrastive` | Same | +| Contrastive modes | info_nce / supcon / grouped | Same | +| Self-contrast | Supported | Same | +| MLM masking | 15% via `DataCollatorForLanguageModeling` | Same | +| Pooling for contrastive | CLS (default) | Mean (recommended) | +| Encoder dispatch | `model.roberta` | `model.model` | +| LM head dispatch | `model.lm_head()` | `model.decoder(model.head())` | + +## Pooling: Mean vs CLS + +ModernBERT uses **mean pooling** by default (`config.classifier_pooling = "mean"`), +not CLS-token pooling. This is a deliberate design choice driven by the architecture: + +- In local attention layers (sliding window of 128 tokens), the CLS token at position 0 + can only attend to tokens within its window. It does **not** see the full sequence. +- Full attention layers (every 3rd) allow global information flow, but the CLS token + still receives a biased view compared to mean pooling over all positions. +- Mean pooling aggregates information from all positions equally, weighted by the + attention mask (padding tokens are excluded). + +For InvPT's contrastive learning, this means: + +``` +CLS pooling: embedding = last_hidden[:, 0, :] +Mean pooling: embedding = (last_hidden * mask).sum(1) / mask.sum(1) +``` + +The `pooling` config field controls this in both `compute_loss` and +`_compute_grouped_loss` of `ContrastiveTrainer`. It is architecture-independent — +you can use mean pooling with RoBERTa too, though CLS is the conventional choice there. + +## Architecture Dispatch + +The `SplitHeadWrapper` auto-detects the model architecture via attribute inspection: + +| Component | RoBERTa | ModernBERT | Detection | +| ---------------- | ----------------------- | ----------------------------------- | ------------------------------- | +| Encoder backbone | `model.roberta` | `model.model` | `hasattr(mlm_model, "roberta")` | +| LM head | `model.lm_head(hidden)` | `model.decoder(model.head(hidden))` | `hasattr(mlm_model, "lm_head")` | + +ModernBERT splits the LM head into two stages: + +1. **`head`**: `ModernBertPredictionHead` — dense layer + GELU activation + layer norm +2. **`decoder`**: `nn.Linear(hidden_size, vocab_size)` — projection to vocabulary + +The wrapper applies these sequentially per-chunk to maintain the memory optimization +(never materializing the full `[N, seq_len, vocab_size]` logit tensor). + +## Tokenizer + +ModernBERT uses a BPE tokenizer (from the OLMo lineage) with: + +- Vocabulary size: 50,368 +- Special tokens: `[CLS]` (50281), `[SEP]` (50282), `[PAD]` (50283), `[MASK]` (50284) +- No `token_type_ids` (unlike BERT/RoBERTa) + +The tokenizer is loaded via `AutoTokenizer.from_pretrained()` and is fully compatible +with InvPT's data pipeline — `DataCollatorForLanguageModeling` works without changes +since it only requires `pad_token_id` and `special_tokens_mask`. + +## Downstream Evaluation + +All 7 downstream tasks support ModernBERT via the `--model_type modernbert` flag. +The `MODEL_CLASSES` dict in each `run.py` maps `"modernbert"` to Auto classes: + +```python +MODEL_CLASSES = { + "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer), + "modernbert": (AutoConfig, AutoModel, AutoTokenizer), +} +``` + +For classification tasks (Defect-detection, Code-classification), `AutoModelForSequenceClassification` +is used instead of `AutoModel`. + +Example downstream usage: + +```bash +cd downstream/Clone-detection-POJ-104 +python ./code/run.py \ + --model_type=modernbert \ + --model_name_or_path=saved_models/InvPT-ModernBERT-base/final \ + --tokenizer_name=saved_models/InvPT-ModernBERT-base/final \ + --do_train --do_test \ + --train_data_file=./dataset/train.jsonl \ + --eval_data_file=./dataset/valid.jsonl \ + --test_data_file=./dataset/test.jsonl \ + --block_size 400 \ + --train_batch_size 8 \ + --eval_batch_size 64 \ + --learning_rate 2e-5 \ + --epoch 2 +``` + +Note: the downstream `model.py` wrappers extract embeddings via `outputs[0][:, 0, :]` +(CLS token) as a fallback when no pooled output is available. `ModernBertModel` returns +`BaseModelOutput` with only `last_hidden_state`, so the CLS fallback is used. This is +acceptable for fine-tuning where the model learns task-specific representations, but +mean pooling may be worth implementing in downstream tasks for consistency. + +## Requirements + +- `transformers >= 4.48.0` (ModernBERT was added in this release) +- `flash-attn` (optional, recommended for GPU efficiency) + +Install: + +```bash +uv add "transformers>=4.48" +pip install flash-attn # optional +``` + +## Saved Model Format + +The pre-trained model is saved as a standard HuggingFace checkpoint: + +```python +unwrapped.mlm_model.save_pretrained(save_path) # saves ModernBertForMaskedLM +tokenizer.save_pretrained(save_path) +``` + +The saved checkpoint is loadable with: + +```python +from transformers import AutoModelForMaskedLM, AutoTokenizer + +model = AutoModelForMaskedLM.from_pretrained("saved_models/InvPT-ModernBERT-base/final") +tokenizer = AutoTokenizer.from_pretrained("saved_models/InvPT-ModernBERT-base/final") +``` + +The `config.json` in the saved directory contains `"model_type": "modernbert"`, so +Auto classes automatically resolve to the correct ModernBERT implementation. diff --git a/downstream/Clone-detection-BigCloneBench/code/run.py b/downstream/Clone-detection-BigCloneBench/code/run.py index 257913d..14a8f03 100644 --- a/downstream/Clone-detection-BigCloneBench/code/run.py +++ b/downstream/Clone-detection-BigCloneBench/code/run.py @@ -50,6 +50,9 @@ cpu_cont = multiprocessing.cpu_count() from transformers import ( AdamW, + AutoConfig, + AutoModel, + AutoTokenizer, BertConfig, BertForMaskedLM, BertTokenizer, @@ -76,6 +79,7 @@ "bert": (BertConfig, BertForMaskedLM, BertTokenizer), "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer), "distilbert": (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer), + "modernbert": (AutoConfig, AutoModel, AutoTokenizer), } diff --git a/downstream/Clone-detection-CodeNet/code/model.py b/downstream/Clone-detection-CodeNet/code/model.py index 2824e0a..a540c3f 100644 --- a/downstream/Clone-detection-CodeNet/code/model.py +++ b/downstream/Clone-detection-CodeNet/code/model.py @@ -14,15 +14,23 @@ def __init__(self, encoder, config, tokenizer, args): self.tokenizer = tokenizer self.args = args + def _pool(self, hidden_states, attention_mask): + """Mean pooling over non-padding tokens.""" + mask = attention_mask.unsqueeze(-1).float() + return (hidden_states * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9) + def forward(self, input_ids=None, p_input_ids=None, n_input_ids=None, labels=None): bs, _ = input_ids.size() input_ids = torch.cat((input_ids, p_input_ids, n_input_ids), 0) + attention_mask = input_ids.ne(self.tokenizer.pad_token_id) - outputs = self.encoder(input_ids, attention_mask=input_ids.ne(1)) - if len(outputs) > 1: - outputs = outputs[1] + encoder_outputs = self.encoder(input_ids, attention_mask=attention_mask) + if hasattr(self.args, "model_type") and self.args.model_type == "modernbert": + outputs = self._pool(encoder_outputs[0], attention_mask) + elif len(encoder_outputs) > 1: + outputs = encoder_outputs[1] else: - outputs = outputs[0][:, 0, :] + outputs = encoder_outputs[0][:, 0, :] outputs = outputs.split(bs, 0) prob_1 = (outputs[0] * outputs[1]).sum(-1) diff --git a/downstream/Clone-detection-CodeNet/code/run.py b/downstream/Clone-detection-CodeNet/code/run.py index 0c6be3a..6b85dea 100644 --- a/downstream/Clone-detection-CodeNet/code/run.py +++ b/downstream/Clone-detection-CodeNet/code/run.py @@ -51,6 +51,9 @@ cpu_cont = multiprocessing.cpu_count() from transformers import ( AdamW, + AutoConfig, + AutoModel, + AutoTokenizer, BertConfig, BertModel, BertTokenizer, @@ -77,6 +80,7 @@ "bert": (BertConfig, BertModel, BertTokenizer), "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer), "distilbert": (DistilBertConfig, DistilBertModel, DistilBertTokenizer), + "modernbert": (AutoConfig, AutoModel, AutoTokenizer), } diff --git a/downstream/Clone-detection-CodeNet/run.sh b/downstream/Clone-detection-CodeNet/run.sh index 9ff115a..d5fbaef 100755 --- a/downstream/Clone-detection-CodeNet/run.sh +++ b/downstream/Clone-detection-CodeNet/run.sh @@ -2,6 +2,8 @@ model_path=$1 save_path=$2 subset=$3 +model_type=${4:-roberta} +tokenizer_name=${5:-roberta-base} output_dir=$save_path/$subset @@ -10,9 +12,9 @@ touch $output_dir/train.log python ./code/run.py \ --output_dir=$output_dir \ - --model_type=roberta \ + --model_type=$model_type \ --model_name_or_path=$model_path \ - --tokenizer_name=roberta-base \ + --tokenizer_name=$tokenizer_name \ --do_train \ --do_test \ --train_data_file=./dataset/$subset/train.jsonl \ @@ -38,4 +40,4 @@ python evaluator/evaluator.py \ echo "Running evaluation for augmented test set..." -./run_aug_test.sh $model_path $save_path $subset +./run_aug_test.sh $model_path $save_path $subset $model_type $tokenizer_name diff --git a/downstream/Clone-detection-CodeNet/run_aug_test.sh b/downstream/Clone-detection-CodeNet/run_aug_test.sh index 515f73d..dc06ce8 100755 --- a/downstream/Clone-detection-CodeNet/run_aug_test.sh +++ b/downstream/Clone-detection-CodeNet/run_aug_test.sh @@ -2,6 +2,8 @@ model_path=$1 save_path=$2 subset=$3 +model_type=${4:-roberta} +tokenizer_name=${5:-roberta-base} output_dir=$save_path/$subset @@ -10,9 +12,9 @@ touch $output_dir/aug_train.log python ./code/run.py \ --output_dir=$output_dir \ - --model_type=roberta \ + --model_type=$model_type \ --model_name_or_path=$model_path \ - --tokenizer_name=roberta-base \ + --tokenizer_name=$tokenizer_name \ --do_test \ --train_data_file=./dataset/$subset/train.jsonl \ --eval_data_file=./dataset/$subset/valid.jsonl \ diff --git a/downstream/Clone-detection-POJ104/code/model.py b/downstream/Clone-detection-POJ104/code/model.py index ecdc7f1..18df82a 100644 --- a/downstream/Clone-detection-POJ104/code/model.py +++ b/downstream/Clone-detection-POJ104/code/model.py @@ -13,15 +13,23 @@ def __init__(self, encoder, config, tokenizer, args): self.tokenizer = tokenizer self.args = args + def _pool(self, hidden_states, attention_mask): + """Mean pooling over non-padding tokens.""" + mask = attention_mask.unsqueeze(-1).float() + return (hidden_states * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9) + def forward(self, input_ids=None, p_input_ids=None, n_input_ids=None, labels=None): bs, _ = input_ids.size() input_ids = torch.cat((input_ids, p_input_ids, n_input_ids), 0) + attention_mask = input_ids.ne(self.tokenizer.pad_token_id) - outputs = self.encoder(input_ids, attention_mask=input_ids.ne(1)) - if len(outputs) > 1: - outputs = outputs[1] + encoder_outputs = self.encoder(input_ids, attention_mask=attention_mask) + if hasattr(self.args, "model_type") and self.args.model_type == "modernbert": + outputs = self._pool(encoder_outputs[0], attention_mask) + elif len(encoder_outputs) > 1: + outputs = encoder_outputs[1] else: - outputs = outputs[0][:, 0, :] + outputs = encoder_outputs[0][:, 0, :] outputs = outputs.split(bs, 0) prob_1 = (outputs[0] * outputs[1]).sum(-1) diff --git a/downstream/Clone-detection-POJ104/code/run.py b/downstream/Clone-detection-POJ104/code/run.py index 0c6be3a..6b85dea 100644 --- a/downstream/Clone-detection-POJ104/code/run.py +++ b/downstream/Clone-detection-POJ104/code/run.py @@ -51,6 +51,9 @@ cpu_cont = multiprocessing.cpu_count() from transformers import ( AdamW, + AutoConfig, + AutoModel, + AutoTokenizer, BertConfig, BertModel, BertTokenizer, @@ -77,6 +80,7 @@ "bert": (BertConfig, BertModel, BertTokenizer), "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer), "distilbert": (DistilBertConfig, DistilBertModel, DistilBertTokenizer), + "modernbert": (AutoConfig, AutoModel, AutoTokenizer), } diff --git a/downstream/Clone-detection-POJ104/run.sh b/downstream/Clone-detection-POJ104/run.sh index f8a5347..5b1830b 100755 --- a/downstream/Clone-detection-POJ104/run.sh +++ b/downstream/Clone-detection-POJ104/run.sh @@ -1,14 +1,17 @@ #!/bin/bash model_path=$1 output_dir=$2 +model_type=${3:-roberta} +tokenizer_name=${4:-roberta-base} + mkdir -p $output_dir touch $output_dir/train.log python ./code/run.py \ --output_dir=$output_dir \ - --model_type=roberta \ + --model_type=$model_type \ --model_name_or_path=$model_path \ - --tokenizer_name=roberta-base \ + --tokenizer_name=$tokenizer_name \ --do_train \ --do_test \ --train_data_file=./dataset/train.jsonl \ @@ -34,4 +37,4 @@ python evaluator/evaluator.py \ echo "Running evaluation for augmented test set..." -./run_aug_test.sh $model_path $output_dir +./run_aug_test.sh $model_path $output_dir $model_type $tokenizer_name diff --git a/downstream/Clone-detection-POJ104/run_aug_test.sh b/downstream/Clone-detection-POJ104/run_aug_test.sh index db4aed4..43c99f6 100755 --- a/downstream/Clone-detection-POJ104/run_aug_test.sh +++ b/downstream/Clone-detection-POJ104/run_aug_test.sh @@ -1,6 +1,8 @@ #!/bin/bash model_path=$1 save_path=$2 +model_type=${3:-roberta} +tokenizer_name=${4:-roberta-base} output_dir=$save_path @@ -9,9 +11,9 @@ touch $output_dir/aug_train.log python ./code/run.py \ --output_dir=$output_dir \ - --model_type=roberta \ + --model_type=$model_type \ --model_name_or_path=$model_path \ - --tokenizer_name=roberta-base \ + --tokenizer_name=$tokenizer_name \ --do_test \ --train_data_file=./dataset/train.jsonl \ --eval_data_file=./dataset/valid.jsonl \ diff --git a/downstream/Code-classification-CodeNet/code/model.py b/downstream/Code-classification-CodeNet/code/model.py index 1cde785..ff9d963 100644 --- a/downstream/Code-classification-CodeNet/code/model.py +++ b/downstream/Code-classification-CodeNet/code/model.py @@ -14,7 +14,9 @@ def __init__(self, encoder, config, tokenizer, args): self.args = args def forward(self, input_ids=None, labels=None): - logits = self.encoder(input_ids, attention_mask=input_ids.ne(1))[0] + logits = self.encoder( + input_ids, attention_mask=input_ids.ne(self.tokenizer.pad_token_id) + )[0] prob = torch.softmax(logits, -1) if labels is not None: loss_fct = nn.CrossEntropyLoss(ignore_index=-1) diff --git a/downstream/Code-classification-CodeNet/code/run.py b/downstream/Code-classification-CodeNet/code/run.py index e367000..8ad0e24 100644 --- a/downstream/Code-classification-CodeNet/code/run.py +++ b/downstream/Code-classification-CodeNet/code/run.py @@ -39,12 +39,20 @@ from tqdm import tqdm from transformers import ( AdamW, + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer, get_linear_schedule_with_warmup, ) +MODEL_CLASSES = { + "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), + "modernbert": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer), +} + logger = logging.getLogger(__name__) @@ -341,6 +349,12 @@ def main(): type=str, help="The model checkpoint for weights initialization.", ) + parser.add_argument( + "--model_type", + default="roberta", + type=str, + help="Model type (roberta or modernbert).", + ) parser.add_argument( "--tokenizer_name", default="", @@ -415,12 +429,13 @@ def main(): # Set seed set_seed(args.seed) - config = RobertaConfig.from_pretrained(args.model_name_or_path) + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.model_name_or_path) config.num_labels = 104 - tokenizer = RobertaTokenizer.from_pretrained(args.tokenizer_name) - model = RobertaForSequenceClassification.from_pretrained( - args.model_name_or_path, config=config + tokenizer = tokenizer_class.from_pretrained( + args.tokenizer_name if args.tokenizer_name else args.model_name_or_path ) + model = model_class.from_pretrained(args.model_name_or_path, config=config) model = Model(model, config, tokenizer, args) diff --git a/downstream/Code-classification-CodeNet/run.sh b/downstream/Code-classification-CodeNet/run.sh index 7809791..2c4d515 100755 --- a/downstream/Code-classification-CodeNet/run.sh +++ b/downstream/Code-classification-CodeNet/run.sh @@ -2,6 +2,8 @@ model_path=$1 save_path=$2 subset=$3 +model_type=${4:-roberta} +tokenizer_name=${5:-microsoft/codebert-base} output_dir=$save_path/$subset @@ -11,7 +13,8 @@ touch $output_dir/test_train.log echo "Running fine-tuning for subset: $subset" python ./code/run.py \ --output_dir=$output_dir \ - --tokenizer_name=microsoft/codebert-base \ + --model_type=$model_type \ + --tokenizer_name=$tokenizer_name \ --model_name_or_path=$model_path \ --do_train \ --do_test \ @@ -28,4 +31,4 @@ python ./code/run.py \ echo "Running evaluation for augmented test set..." -./run_aug_test.sh $model_path $save_path $subset +./run_aug_test.sh $model_path $save_path $subset $model_type $tokenizer_name diff --git a/downstream/Code-classification-CodeNet/run_aug_test.sh b/downstream/Code-classification-CodeNet/run_aug_test.sh index cccafbb..bcfe01f 100755 --- a/downstream/Code-classification-CodeNet/run_aug_test.sh +++ b/downstream/Code-classification-CodeNet/run_aug_test.sh @@ -2,6 +2,8 @@ model_path=$1 save_path=$2 subset=$3 +model_type=${4:-roberta} +tokenizer_name=${5:-microsoft/codebert-base} output_dir=$save_path/$subset @@ -10,7 +12,8 @@ touch $output_dir/test_train.log python ./code/run.py \ --output_dir=$output_dir \ - --tokenizer_name=microsoft/codebert-base \ + --model_type=$model_type \ + --tokenizer_name=$tokenizer_name \ --model_name_or_path=$model_path \ --do_test \ --train_data_file=./dataset/$subset/train.jsonl \ diff --git a/downstream/Code-classification-POJ104/code/model.py b/downstream/Code-classification-POJ104/code/model.py index 1cde785..ff9d963 100644 --- a/downstream/Code-classification-POJ104/code/model.py +++ b/downstream/Code-classification-POJ104/code/model.py @@ -14,7 +14,9 @@ def __init__(self, encoder, config, tokenizer, args): self.args = args def forward(self, input_ids=None, labels=None): - logits = self.encoder(input_ids, attention_mask=input_ids.ne(1))[0] + logits = self.encoder( + input_ids, attention_mask=input_ids.ne(self.tokenizer.pad_token_id) + )[0] prob = torch.softmax(logits, -1) if labels is not None: loss_fct = nn.CrossEntropyLoss(ignore_index=-1) diff --git a/downstream/Code-classification-POJ104/code/run.py b/downstream/Code-classification-POJ104/code/run.py index e367000..8ad0e24 100644 --- a/downstream/Code-classification-POJ104/code/run.py +++ b/downstream/Code-classification-POJ104/code/run.py @@ -39,12 +39,20 @@ from tqdm import tqdm from transformers import ( AdamW, + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer, get_linear_schedule_with_warmup, ) +MODEL_CLASSES = { + "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), + "modernbert": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer), +} + logger = logging.getLogger(__name__) @@ -341,6 +349,12 @@ def main(): type=str, help="The model checkpoint for weights initialization.", ) + parser.add_argument( + "--model_type", + default="roberta", + type=str, + help="Model type (roberta or modernbert).", + ) parser.add_argument( "--tokenizer_name", default="", @@ -415,12 +429,13 @@ def main(): # Set seed set_seed(args.seed) - config = RobertaConfig.from_pretrained(args.model_name_or_path) + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.model_name_or_path) config.num_labels = 104 - tokenizer = RobertaTokenizer.from_pretrained(args.tokenizer_name) - model = RobertaForSequenceClassification.from_pretrained( - args.model_name_or_path, config=config + tokenizer = tokenizer_class.from_pretrained( + args.tokenizer_name if args.tokenizer_name else args.model_name_or_path ) + model = model_class.from_pretrained(args.model_name_or_path, config=config) model = Model(model, config, tokenizer, args) diff --git a/downstream/Code-classification-POJ104/run.sh b/downstream/Code-classification-POJ104/run.sh index e334604..963d538 100755 --- a/downstream/Code-classification-POJ104/run.sh +++ b/downstream/Code-classification-POJ104/run.sh @@ -2,6 +2,8 @@ model_path=$1 save_path=$2 subset=$3 +model_type=${4:-roberta} +tokenizer_name=${5:-microsoft/codebert-base} output_dir=$save_path/$subset @@ -11,7 +13,8 @@ touch $output_dir/test_train.log echo "Running fine-tuning for POJ104" python ./code/run.py \ --output_dir=$output_dir \ - --tokenizer_name=microsoft/codebert-base \ + --model_type=$model_type \ + --tokenizer_name=$tokenizer_name \ --model_name_or_path=$model_path \ --do_train \ --do_test \ @@ -27,4 +30,4 @@ python ./code/run.py \ --seed 123456 2>&1 | tee $output_dir/test_train.log echo "Running evaluation for augmented test set..." -./run_aug_test.sh $model_path $save_path $subset +./run_aug_test.sh $model_path $save_path $subset $model_type $tokenizer_name diff --git a/downstream/Code-classification-POJ104/run_aug_test.sh b/downstream/Code-classification-POJ104/run_aug_test.sh index cccafbb..bcfe01f 100755 --- a/downstream/Code-classification-POJ104/run_aug_test.sh +++ b/downstream/Code-classification-POJ104/run_aug_test.sh @@ -2,6 +2,8 @@ model_path=$1 save_path=$2 subset=$3 +model_type=${4:-roberta} +tokenizer_name=${5:-microsoft/codebert-base} output_dir=$save_path/$subset @@ -10,7 +12,8 @@ touch $output_dir/test_train.log python ./code/run.py \ --output_dir=$output_dir \ - --tokenizer_name=microsoft/codebert-base \ + --model_type=$model_type \ + --tokenizer_name=$tokenizer_name \ --model_name_or_path=$model_path \ --do_test \ --train_data_file=./dataset/$subset/train.jsonl \ diff --git a/downstream/Code-translation/code/run.py b/downstream/Code-translation/code/run.py index eb265dc..909eb18 100644 --- a/downstream/Code-translation/code/run.py +++ b/downstream/Code-translation/code/run.py @@ -36,13 +36,19 @@ from torch.utils.data.distributed import DistributedSampler from transformers import ( AdamW, + AutoConfig, + AutoModel, + AutoTokenizer, get_linear_schedule_with_warmup, RobertaConfig, RobertaModel, RobertaTokenizer, ) -MODEL_CLASSES = {"roberta": (RobertaConfig, RobertaModel, RobertaTokenizer)} +MODEL_CLASSES = { + "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer), + "modernbert": (AutoConfig, AutoModel, AutoTokenizer), +} logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", diff --git a/downstream/Defect-detection/code/run.py b/downstream/Defect-detection/code/run.py index 33b3cd3..096be7b 100644 --- a/downstream/Defect-detection/code/run.py +++ b/downstream/Defect-detection/code/run.py @@ -50,6 +50,9 @@ cpu_cont = multiprocessing.cpu_count() from transformers import ( AdamW, + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, BertConfig, BertForSequenceClassification, BertTokenizer, @@ -75,6 +78,7 @@ "openai-gpt": (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), "bert": (BertConfig, BertForSequenceClassification, BertTokenizer), "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), + "modernbert": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer), "distilbert": ( DistilBertConfig, DistilBertForSequenceClassification, diff --git a/experiments/modernbert_base.yaml b/experiments/modernbert_base.yaml new file mode 100644 index 0000000..ba5336c --- /dev/null +++ b/experiments/modernbert_base.yaml @@ -0,0 +1,5 @@ +model_name: "answerdotai/ModernBERT-base" +model_type: "modernbert" +pooling: "mean" +max_seq_length: 512 +run_name: "InvPT-ModernBERT-base" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_codebert.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_codebert.sh new file mode 100755 index 0000000..f934894 --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_contrabert_c.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_contrabert_c.sh new file mode 100755 index 0000000..3160e0b --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_contrabert_g.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_contrabert_g.sh new file mode 100755 index 0000000..1c83546 --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_graphcodebert.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_graphcodebert.sh new file mode 100755 index 0000000..9421fa4 --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-codebert.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-codebert.sh new file mode 100755 index 0000000..cbcf312 --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-contrabert_c.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-contrabert_c.sh new file mode 100755 index 0000000..f037725 --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-contrabert_g.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-contrabert_g.sh new file mode 100755 index 0000000..4ff8ed8 --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-graphcodebert.sh b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-graphcodebert.sh new file mode 100755 index 0000000..4c6d4a6 --- /dev/null +++ b/experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-BigCloneBench with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Clone-detection-BigCloneBench" diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codebert.sh new file mode 100755 index 0000000..06a7a1a --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Clone-detection-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_contrabert_c.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_contrabert_c.sh new file mode 100755 index 0000000..1ac93ab --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Clone-detection-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_contrabert_g.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_contrabert_g.sh new file mode 100755 index 0000000..9980723 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Clone-detection-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_graphcodebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_graphcodebert.sh new file mode 100755 index 0000000..232b7d7 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Clone-detection-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-codebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-codebert.sh new file mode 100755 index 0000000..1fd8e12 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Clone-detection-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-contrabert_c.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-contrabert_c.sh new file mode 100755 index 0000000..40b89fb --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Clone-detection-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-contrabert_g.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-contrabert_g.sh new file mode 100755 index 0000000..6ad5cce --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Clone-detection-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-graphcodebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-graphcodebert.sh new file mode 100755 index 0000000..82568e4 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (C++1400) with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Clone-detection-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codebert.sh new file mode 100755 index 0000000..f584af0 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Clone-detection-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_contrabert_c.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_contrabert_c.sh new file mode 100755 index 0000000..e2f33f4 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Clone-detection-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_contrabert_g.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_contrabert_g.sh new file mode 100755 index 0000000..5c45ae0 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Clone-detection-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_graphcodebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_graphcodebert.sh new file mode 100755 index 0000000..a228875 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Clone-detection-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-codebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-codebert.sh new file mode 100755 index 0000000..ce83d95 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Clone-detection-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-contrabert_c.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-contrabert_c.sh new file mode 100755 index 0000000..269102c --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Clone-detection-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-contrabert_g.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-contrabert_g.sh new file mode 100755 index 0000000..1591e84 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Clone-detection-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-graphcodebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-graphcodebert.sh new file mode 100755 index 0000000..b298954 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Java250) with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Clone-detection-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_codebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_codebert.sh new file mode 100755 index 0000000..8f4affc --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Clone-detection-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_contrabert_c.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_contrabert_c.sh new file mode 100755 index 0000000..7c7291d --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Clone-detection-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_contrabert_g.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_contrabert_g.sh new file mode 100755 index 0000000..211c493 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Clone-detection-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_graphcodebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_graphcodebert.sh new file mode 100755 index 0000000..926a7fa --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Clone-detection-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-codebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-codebert.sh new file mode 100755 index 0000000..9d684bc --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Clone-detection-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-contrabert_c.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-contrabert_c.sh new file mode 100755 index 0000000..ac08095 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Clone-detection-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-contrabert_g.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-contrabert_g.sh new file mode 100755 index 0000000..c515dc1 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Clone-detection-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-graphcodebert.sh b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-graphcodebert.sh new file mode 100755 index 0000000..05e3cb2 --- /dev/null +++ b/experiments_downstream/Clone-detection-CodeNet/clone-codenet_Python800_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-CodeNet (Python800) with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Clone-detection-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_codebert.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_codebert.sh new file mode 100755 index 0000000..dc50820 --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Clone-detection-POJ104" roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_contrabert_c.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_contrabert_c.sh new file mode 100755 index 0000000..c5c1f1f --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Clone-detection-POJ104" roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_contrabert_g.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_contrabert_g.sh new file mode 100755 index 0000000..f481c1c --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Clone-detection-POJ104" roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_graphcodebert.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_graphcodebert.sh new file mode 100755 index 0000000..a154952 --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Clone-detection-POJ104" roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-codebert.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-codebert.sh new file mode 100755 index 0000000..953b883 --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Clone-detection-POJ104" roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-contrabert_c.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-contrabert_c.sh new file mode 100755 index 0000000..9be34d9 --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Clone-detection-POJ104" roberta microsoft/codebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-contrabert_g.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-contrabert_g.sh new file mode 100755 index 0000000..ad23aa4 --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Clone-detection-POJ104" roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-graphcodebert.sh b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-graphcodebert.sh new file mode 100755 index 0000000..ec5fff5 --- /dev/null +++ b/experiments_downstream/Clone-detection-POJ104/clone-poj104_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Clone-detection-POJ104 with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Clone-detection-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Clone-detection-POJ104" roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_codebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_codebert.sh new file mode 100755 index 0000000..a7098eb --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Code-classification-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_contrabert_c.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_contrabert_c.sh new file mode 100755 index 0000000..5f25b48 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Code-classification-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_contrabert_g.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_contrabert_g.sh new file mode 100755 index 0000000..77eb658 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Code-classification-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_graphcodebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_graphcodebert.sh new file mode 100755 index 0000000..420b007 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Code-classification-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-codebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-codebert.sh new file mode 100755 index 0000000..2552f68 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Code-classification-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-contrabert_c.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-contrabert_c.sh new file mode 100755 index 0000000..9ff69f6 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Code-classification-CodeNet" C++1400 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-contrabert_g.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-contrabert_g.sh new file mode 100755 index 0000000..8d5df3c --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Code-classification-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-graphcodebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-graphcodebert.sh new file mode 100755 index 0000000..caed086 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_C++1400_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (C++1400) with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Code-classification-CodeNet" C++1400 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_codebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_codebert.sh new file mode 100755 index 0000000..8e7ea16 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Code-classification-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_contrabert_c.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_contrabert_c.sh new file mode 100755 index 0000000..4f27974 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Code-classification-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_contrabert_g.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_contrabert_g.sh new file mode 100755 index 0000000..0fd50b1 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Code-classification-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_graphcodebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_graphcodebert.sh new file mode 100755 index 0000000..fe99ed7 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Code-classification-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-codebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-codebert.sh new file mode 100755 index 0000000..69e8108 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Code-classification-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-contrabert_c.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-contrabert_c.sh new file mode 100755 index 0000000..8103c60 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Code-classification-CodeNet" Java250 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-contrabert_g.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-contrabert_g.sh new file mode 100755 index 0000000..1bbfb4a --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Code-classification-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-graphcodebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-graphcodebert.sh new file mode 100755 index 0000000..97a648c --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Java250_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Java250) with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Code-classification-CodeNet" Java250 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_codebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_codebert.sh new file mode 100755 index 0000000..39fb03c --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Code-classification-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_contrabert_c.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_contrabert_c.sh new file mode 100755 index 0000000..1aa25b7 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Code-classification-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_contrabert_g.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_contrabert_g.sh new file mode 100755 index 0000000..f1cef41 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Code-classification-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_graphcodebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_graphcodebert.sh new file mode 100755 index 0000000..aca03e6 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Code-classification-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-codebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-codebert.sh new file mode 100755 index 0000000..a8f6379 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Code-classification-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-contrabert_c.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-contrabert_c.sh new file mode 100755 index 0000000..30269f6 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Code-classification-CodeNet" Python800 roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-contrabert_g.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-contrabert_g.sh new file mode 100755 index 0000000..f8f6f08 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Code-classification-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-graphcodebert.sh b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-graphcodebert.sh new file mode 100755 index 0000000..b67e352 --- /dev/null +++ b/experiments_downstream/Code-classification-CodeNet/cls-codenet_Python800_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-CodeNet (Python800) with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-CodeNet" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Code-classification-CodeNet" Python800 roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_codebert.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_codebert.sh new file mode 100755 index 0000000..f406cf7 --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Code-classification-POJ104" Cpp roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_contrabert_c.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_contrabert_c.sh new file mode 100755 index 0000000..f9fa75e --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Code-classification-POJ104" Cpp roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_contrabert_g.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_contrabert_g.sh new file mode 100755 index 0000000..a87fb4f --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Code-classification-POJ104" Cpp roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_graphcodebert.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_graphcodebert.sh new file mode 100755 index 0000000..aa05ab0 --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Code-classification-POJ104" Cpp roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-codebert.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-codebert.sh new file mode 100755 index 0000000..0306d6f --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Code-classification-POJ104" Cpp roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-contrabert_c.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-contrabert_c.sh new file mode 100755 index 0000000..71a2807 --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Code-classification-POJ104" Cpp roberta microsoft/codebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-contrabert_g.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-contrabert_g.sh new file mode 100755 index 0000000..48df121 --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Code-classification-POJ104" Cpp roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-graphcodebert.sh b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-graphcodebert.sh new file mode 100755 index 0000000..db645bd --- /dev/null +++ b/experiments_downstream/Code-classification-POJ104/cls-poj104_Cpp_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-classification-POJ104 (Cpp) with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-classification-POJ104" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Code-classification-POJ104" Cpp roberta microsoft/graphcodebert-base diff --git a/experiments_downstream/Code-translation/translation_codebert.sh b/experiments_downstream/Code-translation/translation_codebert.sh new file mode 100755 index 0000000..4e1156a --- /dev/null +++ b/experiments_downstream/Code-translation/translation_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Code-translation" diff --git a/experiments_downstream/Code-translation/translation_contrabert_c.sh b/experiments_downstream/Code-translation/translation_contrabert_c.sh new file mode 100755 index 0000000..35eb1cf --- /dev/null +++ b/experiments_downstream/Code-translation/translation_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Code-translation" diff --git a/experiments_downstream/Code-translation/translation_contrabert_g.sh b/experiments_downstream/Code-translation/translation_contrabert_g.sh new file mode 100755 index 0000000..b273b76 --- /dev/null +++ b/experiments_downstream/Code-translation/translation_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Code-translation" diff --git a/experiments_downstream/Code-translation/translation_graphcodebert.sh b/experiments_downstream/Code-translation/translation_graphcodebert.sh new file mode 100755 index 0000000..9a61086 --- /dev/null +++ b/experiments_downstream/Code-translation/translation_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Code-translation" diff --git a/experiments_downstream/Code-translation/translation_inv-codebert.sh b/experiments_downstream/Code-translation/translation_inv-codebert.sh new file mode 100755 index 0000000..91621d0 --- /dev/null +++ b/experiments_downstream/Code-translation/translation_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Code-translation" diff --git a/experiments_downstream/Code-translation/translation_inv-contrabert_c.sh b/experiments_downstream/Code-translation/translation_inv-contrabert_c.sh new file mode 100755 index 0000000..4aa7dfa --- /dev/null +++ b/experiments_downstream/Code-translation/translation_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Code-translation" diff --git a/experiments_downstream/Code-translation/translation_inv-contrabert_g.sh b/experiments_downstream/Code-translation/translation_inv-contrabert_g.sh new file mode 100755 index 0000000..5000dc1 --- /dev/null +++ b/experiments_downstream/Code-translation/translation_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Code-translation" diff --git a/experiments_downstream/Code-translation/translation_inv-graphcodebert.sh b/experiments_downstream/Code-translation/translation_inv-graphcodebert.sh new file mode 100755 index 0000000..759ca8a --- /dev/null +++ b/experiments_downstream/Code-translation/translation_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Code-translation with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Code-translation" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Code-translation" diff --git a/experiments_downstream/Defect-detection/defect_codebert.sh b/experiments_downstream/Defect-detection/defect_codebert.sh new file mode 100755 index 0000000..f9d68b2 --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh microsoft/codebert-base "$ROOT_DIR/results/codebert/Defect-detection" diff --git a/experiments_downstream/Defect-detection/defect_contrabert_c.sh b/experiments_downstream/Defect-detection/defect_contrabert_c.sh new file mode 100755 index 0000000..fbd9e5c --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_C" "$ROOT_DIR/results/contrabert_c/Defect-detection" diff --git a/experiments_downstream/Defect-detection/defect_contrabert_g.sh b/experiments_downstream/Defect-detection/defect_contrabert_g.sh new file mode 100755 index 0000000..57caab6 --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh "$ROOT_DIR/saved_models/ContraBERT_G" "$ROOT_DIR/results/contrabert_g/Defect-detection" diff --git a/experiments_downstream/Defect-detection/defect_graphcodebert.sh b/experiments_downstream/Defect-detection/defect_graphcodebert.sh new file mode 100755 index 0000000..3ef89f9 --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh microsoft/graphcodebert-base "$ROOT_DIR/results/graphcodebert/Defect-detection" diff --git a/experiments_downstream/Defect-detection/defect_inv-codebert.sh b/experiments_downstream/Defect-detection/defect_inv-codebert.sh new file mode 100755 index 0000000..930e26f --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_inv-codebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with inv-codebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh "$ROOT_DIR/saved_models/InvCodeBERT-supcon" "$ROOT_DIR/results/inv-codebert/Defect-detection" diff --git a/experiments_downstream/Defect-detection/defect_inv-contrabert_c.sh b/experiments_downstream/Defect-detection/defect_inv-contrabert_c.sh new file mode 100755 index 0000000..fdc2c5a --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_inv-contrabert_c.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with inv-contrabert_c +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_C-supcon" "$ROOT_DIR/results/inv-contrabert_c/Defect-detection" diff --git a/experiments_downstream/Defect-detection/defect_inv-contrabert_g.sh b/experiments_downstream/Defect-detection/defect_inv-contrabert_g.sh new file mode 100755 index 0000000..390d8db --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_inv-contrabert_g.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with inv-contrabert_g +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh "$ROOT_DIR/saved_models/InvContraBERT_G-supcon" "$ROOT_DIR/results/inv-contrabert_g/Defect-detection" diff --git a/experiments_downstream/Defect-detection/defect_inv-graphcodebert.sh b/experiments_downstream/Defect-detection/defect_inv-graphcodebert.sh new file mode 100755 index 0000000..1b40338 --- /dev/null +++ b/experiments_downstream/Defect-detection/defect_inv-graphcodebert.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Downstream evaluation: Defect-detection with inv-graphcodebert +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" + +cd "$ROOT_DIR/downstream/Defect-detection" +./run.sh "$ROOT_DIR/saved_models/InvGraphCodeBERT-supcon" "$ROOT_DIR/results/inv-graphcodebert/Defect-detection" diff --git a/experiments_downstream/gen_all.py b/experiments_downstream/gen_all.py new file mode 100644 index 0000000..3b54ad4 --- /dev/null +++ b/experiments_downstream/gen_all.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Generate downstream evaluation scripts for all model+task combinations.""" + +import stat +from pathlib import Path + +BASE = Path(__file__).parent + +models = [ + # (short_name, model_path, tokenizer, model_type) + # Baselines + ("codebert", "microsoft/codebert-base", "microsoft/codebert-base", "roberta"), + ( + "graphcodebert", + "microsoft/graphcodebert-base", + "microsoft/graphcodebert-base", + "roberta", + ), + ( + "contrabert_c", + "./saved_models/ContraBERT_C", + "microsoft/codebert-base", + "roberta", + ), + ( + "contrabert_g", + "./saved_models/ContraBERT_G", + "microsoft/graphcodebert-base", + "roberta", + ), + # Our trained models + ( + "inv-codebert", + "./saved_models/InvCodeBERT-supcon", + "microsoft/codebert-base", + "roberta", + ), + ( + "inv-graphcodebert", + "./saved_models/InvGraphCodeBERT-supcon", + "microsoft/graphcodebert-base", + "roberta", + ), + ( + "inv-contrabert_c", + "./saved_models/InvContraBERT_C-supcon", + "microsoft/codebert-base", + "roberta", + ), + ( + "inv-contrabert_g", + "./saved_models/InvContraBERT_G-supcon", + "microsoft/graphcodebert-base", + "roberta", + ), +] + +# These are the literal shell lines we want in the output. +# Using chr(36) to produce '$' so no shell can possibly interpret them. +D = chr(36) # dollar sign +SD_LINE = f'SCRIPT_DIR="{D}(cd "{D}(dirname "{D}{{BASH_SOURCE[0]}}")" && pwd)"' +RD_LINE = f'ROOT_DIR="{D}(cd "{D}SCRIPT_DIR/../.." && pwd)"' + + +def model_ref(model_path: str) -> str: + """Return the shell expression for the model path.""" + if model_path.startswith("./"): + return f'"{D}ROOT_DIR/{model_path[2:]}"' + return model_path + + +def make_script( + task_dir: str, + short: str, + mpath: str, + subset: str | None = None, + args_extra: str = "", +) -> str: + mr = model_ref(mpath) + op = f'"{D}ROOT_DIR/results/{short}/{task_dir}"' + sl = f" ({subset})" if subset else "" + return "\n".join( + [ + "#!/bin/bash", + f"# Downstream evaluation: {task_dir}{sl} with {short}", + "set -euo pipefail", + SD_LINE, + RD_LINE, + "", + f'cd "{D}ROOT_DIR/downstream/{task_dir}"', + f"./run.sh {mr} {op}{args_extra}", + "", + ] + ) + + +def write_script(task_dir: str, fname: str, content: str) -> None: + dp = BASE / task_dir + dp.mkdir(parents=True, exist_ok=True) + fp = dp / fname + fp.write_text(content) + fp.chmod(fp.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +count = 0 +for short, mpath, tok, mtype in models: + # Clone-detection-POJ104: run.sh + write_script( + "Clone-detection-POJ104", + f"clone-poj104_{short}.sh", + make_script( + "Clone-detection-POJ104", short, mpath, args_extra=f" {mtype} {tok}" + ), + ) + count += 1 + + # Clone-detection-CodeNet: run.sh + for sub in ["Java250", "Python800", "C++1400"]: + write_script( + "Clone-detection-CodeNet", + f"clone-codenet_{sub}_{short}.sh", + make_script( + "Clone-detection-CodeNet", + short, + mpath, + subset=sub, + args_extra=f" {sub} {mtype} {tok}", + ), + ) + count += 1 + + # Clone-detection-BigCloneBench: run.sh + write_script( + "Clone-detection-BigCloneBench", + f"clone-bcb_{short}.sh", + make_script("Clone-detection-BigCloneBench", short, mpath), + ) + count += 1 + + # Code-classification-POJ104: run.sh + write_script( + "Code-classification-POJ104", + f"cls-poj104_Cpp_{short}.sh", + make_script( + "Code-classification-POJ104", + short, + mpath, + subset="Cpp", + args_extra=f" Cpp {mtype} {tok}", + ), + ) + count += 1 + + # Code-classification-CodeNet: run.sh + for sub in ["Java250", "Python800", "C++1400"]: + write_script( + "Code-classification-CodeNet", + f"cls-codenet_{sub}_{short}.sh", + make_script( + "Code-classification-CodeNet", + short, + mpath, + subset=sub, + args_extra=f" {sub} {mtype} {tok}", + ), + ) + count += 1 + + # Defect-detection: run.sh + write_script( + "Defect-detection", + f"defect_{short}.sh", + make_script("Defect-detection", short, mpath), + ) + count += 1 + + # Code-translation: run.sh + write_script( + "Code-translation", + f"translation_{short}.sh", + make_script("Code-translation", short, mpath), + ) + count += 1 + +print(f"Generated {count} scripts in {BASE}") diff --git a/modeling/_types.py b/modeling/_types.py index 24e51f7..fff7ceb 100644 --- a/modeling/_types.py +++ b/modeling/_types.py @@ -5,3 +5,8 @@ class ContraMode(str, Enum): INFO_NCE = "info_nce" SUPCON = "supcon" GROUPED = "grouped" + + +class ModelType(str, Enum): + ROBERTA = "roberta" + MODERNBERT = "modernbert" diff --git a/modeling/cli.py b/modeling/cli.py index 97c548a..09fad08 100644 --- a/modeling/cli.py +++ b/modeling/cli.py @@ -4,7 +4,7 @@ import typer -from modeling._types import ContraMode +from modeling._types import ContraMode, ModelType from modeling.common import default_num_proc from modeling.config import load_config from modeling.pretrain import main @@ -102,6 +102,13 @@ def pretrain( help="Use self-contrast (same code, different MLM masks) for rows without augmentation. If disabled, rows without augmentation are dropped." ), ] = True, + model_type: Annotated[ + ModelType, typer.Option(help="Model architecture type.") + ] = ModelType.ROBERTA, + pooling: Annotated[ + str, + typer.Option(help="Pooling strategy for contrastive embeddings (cls or mean)."), + ] = "cls", ) -> None: """Run pre-training with all parameters specified as CLI options. @@ -128,6 +135,8 @@ def pretrain( contra_mode=contra_mode, max_num_augs=max_num_augs, self_contrast=self_contrast, + model_type=model_type, + pooling=pooling, ) diff --git a/modeling/config.py b/modeling/config.py index f24b365..7f84ec2 100644 --- a/modeling/config.py +++ b/modeling/config.py @@ -4,10 +4,10 @@ import dacite import yaml -from ._types import ContraMode +from ._types import ContraMode, ModelType from .common import default_num_proc -_DACITE_CONFIG = dacite.Config(cast=[ContraMode]) +_DACITE_CONFIG = dacite.Config(cast=[ContraMode, ModelType]) @dataclass @@ -33,6 +33,8 @@ class PretrainConfig: contra_mode: ContraMode = ContraMode.INFO_NCE max_num_augs: int = 6 self_contrast: bool = True + model_type: ModelType = ModelType.ROBERTA + pooling: str = "cls" def load_config(path: str | Path) -> PretrainConfig: diff --git a/modeling/model.py b/modeling/model.py index 1cb240e..5057451 100644 --- a/modeling/model.py +++ b/modeling/model.py @@ -1,15 +1,19 @@ import torch import torch.nn as nn import torch.nn.functional as F -from transformers import RobertaForMaskedLM, Trainer +from transformers import Trainer from ._types import ContraMode class SplitHeadWrapper(nn.Module): - """Wraps ``RobertaForMaskedLM`` so that the LM head is applied per-chunk. + """Wraps a ``*ForMaskedLM`` model so the LM head is applied per-chunk. - ``RobertaForMaskedLM.forward()`` always materializes a + Supports both RoBERTa (``RobertaForMaskedLM``) and ModernBERT + (``ModernBertForMaskedLM``). The wrapper auto-detects the encoder and + LM-head attributes. + + The ``*ForMaskedLM.forward()`` always materializes a ``[N, seq_len, vocab_size]`` logit tensor. When the input is a concatenated code+aug batch (``N = 2B`` or ``N = B + B*K``), this doubles/multiplies peak GPU memory and causes OOM. @@ -23,17 +27,34 @@ class SplitHeadWrapper(nn.Module): participate in the single ``forward()`` and gradient sync works normally. """ - def __init__(self, roberta_mlm: RobertaForMaskedLM): + def __init__(self, mlm_model: nn.Module): super().__init__() - self.roberta_mlm = roberta_mlm + self.mlm_model = mlm_model @property def config(self): - return self.roberta_mlm.config + return self.mlm_model.config @property def device(self): - return self.roberta_mlm.device + return self.mlm_model.device + + def _get_encoder(self) -> nn.Module: + """Return the encoder backbone (RobertaModel or ModernBertModel).""" + if hasattr(self.mlm_model, "roberta"): + return self.mlm_model.roberta + if hasattr(self.mlm_model, "model"): + return self.mlm_model.model + raise ValueError(f"Cannot find encoder in {type(self.mlm_model)}") + + def _apply_lm_head(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Apply the LM head to get logits.""" + if hasattr(self.mlm_model, "lm_head"): + return self.mlm_model.lm_head(hidden_states) # RoBERTa + if hasattr(self.mlm_model, "decoder"): + # ModernBERT: prediction head + decoder projection + return self.mlm_model.decoder(self.mlm_model.head(hidden_states)) + raise ValueError(f"Cannot find LM head in {type(self.mlm_model)}") def forward( self, @@ -58,7 +79,8 @@ def forward( ``(mlm_loss, last_hidden_state)`` where ``mlm_loss`` is the average of the per-chunk MLM losses. """ - encoder_outputs = self.roberta_mlm.roberta( + encoder = self._get_encoder() + encoder_outputs = encoder( input_ids=input_ids, attention_mask=attention_mask, output_hidden_states=output_hidden_states, @@ -68,7 +90,7 @@ def forward( last_hidden = encoder_outputs.last_hidden_state # [N, seq, D] # Run lm_head per-chunk to avoid [N, seq, vocab] peak memory. - logits_a = self.roberta_mlm.lm_head(last_hidden[:split_at]) + logits_a = self._apply_lm_head(last_hidden[:split_at]) mlm_loss_a = F.cross_entropy( logits_a.view(-1, logits_a.size(-1)), labels_a.view(-1), @@ -76,7 +98,7 @@ def forward( ) del logits_a - logits_b = self.roberta_mlm.lm_head(last_hidden[split_at:]) + logits_b = self._apply_lm_head(last_hidden[split_at:]) mlm_loss_b = F.cross_entropy( logits_b.view(-1, logits_b.size(-1)), labels_b.view(-1), @@ -304,12 +326,39 @@ class ContrastiveTrainer(Trainer): """ def __init__( - self, alpha=1.0, temperature=0.07, contra_mode="info_nce", *args, **kwargs + self, + alpha=1.0, + temperature=0.07, + contra_mode="info_nce", + pooling="cls", + *args, + **kwargs, ): super().__init__(*args, **kwargs) self.alpha = alpha self.temperature = temperature self.contra_mode = ContraMode(contra_mode) + self.pooling = pooling + + def _pool( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + ) -> torch.Tensor: + """Pool token-level hidden states into a single embedding. + + Args: + hidden_states: ``[N, seq_len, D]``. + attention_mask: ``[N, seq_len]``. + + Returns: + ``[N, D]`` pooled embeddings. + """ + if self.pooling == "mean": + mask = attention_mask.unsqueeze(-1).float() # [N, seq, 1] + return (hidden_states * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-9) + # Default: CLS token + return hidden_states[:, 0, :] def compute_loss( self, model, inputs, return_outputs=False, num_items_in_batch=None @@ -340,8 +389,8 @@ def compute_loss( split_at=B, ) - code_embeddings = last_hidden[:B, 0, :] - aug_embeddings = last_hidden[B:, 0, :] + code_embeddings = self._pool(last_hidden[:B], code_attention_mask) + aug_embeddings = self._pool(last_hidden[B:], aug_attention_mask) # Compute contrastive loss between code and its augmentation if self.contra_mode == ContraMode.SUPCON: @@ -394,8 +443,8 @@ def _compute_grouped_loss(self, model, inputs, return_outputs=False): split_at=B, ) - code_embeddings = last_hidden[:B, 0, :] - aug_embeddings = last_hidden[B:, 0, :] + code_embeddings = self._pool(last_hidden[:B], code_attention_mask) + aug_embeddings = self._pool(last_hidden[B:], aug_attention_mask) # Contrastive loss contrastive_loss = grouped_contrastive_loss( @@ -417,8 +466,8 @@ def prediction_step(self, model, inputs, prediction_loss_only, ignore_keys=None) # For eval we only need code (no aug), so both halves are the # same chunk. Pass an empty second chunk. base = model.module if hasattr(model, "module") else model - roberta_mlm = base.roberta_mlm - outputs = roberta_mlm( + mlm_model = base.mlm_model + outputs = mlm_model( input_ids=code_input_ids, attention_mask=code_attention_mask, labels=code_labels, diff --git a/modeling/pretrain.py b/modeling/pretrain.py index 38ee846..56d17d7 100644 --- a/modeling/pretrain.py +++ b/modeling/pretrain.py @@ -8,10 +8,10 @@ from accelerate import PartialState from datasets import Dataset, Features, Value, load_dataset from transformers import ( + AutoConfig, + AutoModelForMaskedLM, + AutoTokenizer, DataCollatorForLanguageModeling, - RobertaConfig, - RobertaForMaskedLM, - RobertaTokenizerFast, TrainingArguments, ) @@ -224,6 +224,8 @@ def main( contra_mode: ContraMode = "info_nce", max_num_augs: int = 6, self_contrast: bool = True, + model_type: str = "roberta", + pooling: str = "cls", ): set_seed(seed) @@ -239,18 +241,17 @@ def main( os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") tokenizer_name = tokenizer_name or model_name - tokenizer = RobertaTokenizerFast.from_pretrained(tokenizer_name) - config = RobertaConfig.from_pretrained(tokenizer_name) - # model = RobertaForMaskedLM.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(tokenizer_name) + config = AutoConfig.from_pretrained(tokenizer_name) - roberta_mlm = RobertaForMaskedLM.from_pretrained( + mlm_model = AutoModelForMaskedLM.from_pretrained( model_name if checkpoint is None else checkpoint, config=config, - ) # load weights from stage 1 + ) # Wrap so the LM head is applied per-chunk (avoids [2B, seq, vocab] OOM). # DDP will wrap SplitHeadWrapper, keeping all params in one forward(). - model = SplitHeadWrapper(roberta_mlm) + model = SplitHeadWrapper(mlm_model) features = Features( { @@ -353,15 +354,16 @@ def main( alpha=alpha, temperature=temperature, contra_mode=contra_mode, + pooling=pooling, ) trainer.train(resume_from_checkpoint=resume) - # Save the inner RobertaForMaskedLM so downstream tasks can load it - # directly with RobertaForMaskedLM.from_pretrained(). + # Save the inner *ForMaskedLM so downstream tasks can load it + # directly with AutoModelForMaskedLM.from_pretrained(). save_path = f"saved_models/{run_name}/final" unwrapped = trainer.model if hasattr(unwrapped, "module"): # DDP unwrapped = unwrapped.module - unwrapped.roberta_mlm.save_pretrained(save_path) + unwrapped.mlm_model.save_pretrained(save_path) tokenizer.save_pretrained(save_path) diff --git a/tests/test_model_wrapper.py b/tests/test_model_wrapper.py new file mode 100644 index 0000000..bcd23fc --- /dev/null +++ b/tests/test_model_wrapper.py @@ -0,0 +1,203 @@ +"""Tests for SplitHeadWrapper architecture dispatch and pooling.""" + +import pytest +import torch +from transformers import RobertaConfig, RobertaForMaskedLM + +from modeling.model import ContrastiveTrainer, SplitHeadWrapper + +# --------------------------------------------------------------------------- +# SplitHeadWrapper with RoBERTa +# --------------------------------------------------------------------------- + + +def _make_roberta_wrapper( + vocab_size: int = 64, + hidden_size: int = 32, + num_hidden_layers: int = 2, + num_attention_heads: int = 2, + intermediate_size: int = 64, + max_position_embeddings: int = 32, +) -> SplitHeadWrapper: + config = RobertaConfig( + vocab_size=vocab_size, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + max_position_embeddings=max_position_embeddings, + ) + mlm = RobertaForMaskedLM(config) + return SplitHeadWrapper(mlm) + + +class TestSplitHeadWrapperRoberta: + def test_forward_produces_loss_and_hidden(self) -> None: + wrapper = _make_roberta_wrapper() + B, seq = 2, 8 + input_ids = torch.randint(0, 64, (2 * B, seq)) + attention_mask = torch.ones_like(input_ids) + labels_a = torch.randint(0, 64, (B, seq)) + labels_b = torch.randint(0, 64, (B, seq)) + + mlm_loss, last_hidden = wrapper( + input_ids=input_ids, + attention_mask=attention_mask, + labels_a=labels_a, + labels_b=labels_b, + split_at=B, + ) + assert mlm_loss.shape == () + assert last_hidden.shape == (2 * B, seq, 32) + + def test_encoder_dispatch(self) -> None: + wrapper = _make_roberta_wrapper() + encoder = wrapper._get_encoder() + assert encoder is wrapper.mlm_model.roberta + + def test_lm_head_dispatch(self) -> None: + wrapper = _make_roberta_wrapper() + dummy = torch.randn(1, 4, 32) + logits = wrapper._apply_lm_head(dummy) + assert logits.shape == (1, 4, 64) # vocab_size=64 + + +# --------------------------------------------------------------------------- +# SplitHeadWrapper with ModernBERT (skipped if transformers < 4.48) +# --------------------------------------------------------------------------- + +_has_modernbert = True +try: + from transformers import ModernBertConfig, ModernBertForMaskedLM +except ImportError: + _has_modernbert = False + +requires_modernbert = pytest.mark.skipif( + not _has_modernbert, + reason="ModernBERT requires transformers >= 4.48", +) + + +def _make_modernbert_wrapper( + vocab_size: int = 64, + hidden_size: int = 32, + num_hidden_layers: int = 2, + num_attention_heads: int = 2, + intermediate_size: int = 64, + max_position_embeddings: int = 32, +) -> SplitHeadWrapper: + config = ModernBertConfig( + vocab_size=vocab_size, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + max_position_embeddings=max_position_embeddings, + ) + mlm = ModernBertForMaskedLM(config) + return SplitHeadWrapper(mlm) + + +@requires_modernbert +class TestSplitHeadWrapperModernBert: + def test_forward_produces_loss_and_hidden(self) -> None: + wrapper = _make_modernbert_wrapper() + B, seq = 2, 8 + input_ids = torch.randint(0, 64, (2 * B, seq)) + attention_mask = torch.ones_like(input_ids) + labels_a = torch.randint(0, 64, (B, seq)) + labels_b = torch.randint(0, 64, (B, seq)) + + mlm_loss, last_hidden = wrapper( + input_ids=input_ids, + attention_mask=attention_mask, + labels_a=labels_a, + labels_b=labels_b, + split_at=B, + ) + assert mlm_loss.shape == () + assert last_hidden.shape == (2 * B, seq, 32) + + def test_encoder_dispatch(self) -> None: + wrapper = _make_modernbert_wrapper() + encoder = wrapper._get_encoder() + assert encoder is wrapper.mlm_model.model + + def test_lm_head_dispatch(self) -> None: + wrapper = _make_modernbert_wrapper() + dummy = torch.randn(1, 4, 32) + logits = wrapper._apply_lm_head(dummy) + assert logits.shape == (1, 4, 64) # vocab_size=64 + + +# --------------------------------------------------------------------------- +# Pooling +# --------------------------------------------------------------------------- + + +class TestPooling: + def test_cls_pooling(self) -> None: + """CLS pooling extracts the first token.""" + trainer = ContrastiveTrainer.__new__(ContrastiveTrainer) + trainer.pooling = "cls" + hidden = torch.randn(3, 10, 32) + mask = torch.ones(3, 10) + pooled = trainer._pool(hidden, mask) + assert pooled.shape == (3, 32) + assert torch.equal(pooled, hidden[:, 0, :]) + + def test_mean_pooling(self) -> None: + """Mean pooling averages over non-padding tokens.""" + trainer = ContrastiveTrainer.__new__(ContrastiveTrainer) + trainer.pooling = "mean" + hidden = torch.ones(2, 4, 8) # all ones + mask = torch.tensor([[1, 1, 1, 0], [1, 1, 0, 0]], dtype=torch.long) + pooled = trainer._pool(hidden, mask) + assert pooled.shape == (2, 8) + # All non-padding tokens are 1, so mean should be 1 + assert torch.allclose(pooled, torch.ones(2, 8)) + + def test_mean_pooling_ignores_padding(self) -> None: + """Padding tokens (mask=0) should not affect mean pooling.""" + trainer = ContrastiveTrainer.__new__(ContrastiveTrainer) + trainer.pooling = "mean" + hidden = torch.zeros(1, 4, 2) + hidden[0, 0, :] = 2.0 # only first token has value + hidden[0, 1, :] = 4.0 # second token + hidden[0, 2, :] = 99.0 # padding — should be ignored + hidden[0, 3, :] = 99.0 # padding — should be ignored + mask = torch.tensor([[1, 1, 0, 0]]) + pooled = trainer._pool(hidden, mask) + # mean of [2, 4] = 3 + assert torch.allclose(pooled, torch.tensor([[3.0, 3.0]])) + + +# --------------------------------------------------------------------------- +# Config loading with new fields +# --------------------------------------------------------------------------- + + +class TestConfigModelType: + def test_default_model_type(self) -> None: + from modeling._types import ModelType + from modeling.config import PretrainConfig + + cfg = PretrainConfig() + assert cfg.model_type == ModelType.ROBERTA + assert cfg.pooling == "cls" + + def test_modernbert_config_from_yaml(self) -> None: + import tempfile + + import yaml + + from modeling._types import ModelType + from modeling.config import load_config + + data = {"model_type": "modernbert", "pooling": "mean"} + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(data, f) + path = f.name + cfg = load_config(path) + assert cfg.model_type == ModelType.MODERNBERT + assert cfg.pooling == "mean"