Initial Setup

Code Container Testing Steps

1. Preparation: Prepare translation generasion files

# Import required libraries
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
import time
from datetime import datetime
from human_eval.data import write_jsonl, read_problems

# Configure model and tokenizer paths
model_name = "/home/runner/pb_model_finetune/firefly/script/checkpoint/codellama-7b-qlora-sft-merge-v3/"
tokenizer_name = "/home/runner/pb_model_finetune/firefly/script/checkpoint/codellama-7b-qlora-sft-merge-v3/"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = AutoModelForCausalLM.from_pretrained(model_name).to(device)

# Load model and tokenizer
model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name).to(device)

# Set generation parameters
top_p = 0.9
temperature = 0.35
repetition_penalty = 1.0
top_k = 50

# Define function to generate responses

def generate_response(prompt):
    model_input = tokenizer(prompt, return_tensors="pt").to(device)
    model.eval()
    with torch.no_grad():
        return tokenizer.decode(model.generate(**model_input, max_new_tokens=1024, do_sample=True,
        temperature=temperature,
        top_p=top_p, top_k=top_k, repetition_penalty=repetition_penalty)[0], skip_special_tokens=True)

# Read problems
problems = read_problems('/workspace/CodeGeeX/codegeex/benchmark/humaneval-x/java/data/humaneval_java.jsonl.gz')

# Define function to generate completions

def generate_one_completion(task_id, prompt):
    print(datetime.now().strftime("%H:%M:%S"), task_id)
    try:
        result = generate_response(prompt)
        print(result)
        return result
    except:
        print(f"Exception occurs, wait 3 seconds then retry...")
        time.sleep(3)
        generate_one_completion(task_id, prompt)

# Generate responses for each problem and write results to file
num_samples_per_task = 1
for task_id in problems:
    for _ in range(num_samples_per_task):
        samples = [
            dict(
                task_id=task_id,
                generation=generate_one_completion(task_id, problems[task_id]["prompt"]),
                canonical_solution=problems[task_id]["canonical_solution"],
                declaration=problems[task_id]["declaration"],
                example_test=problems[task_id]["example_test"],
                prompt=problems[task_id]["prompt"],
                test=problems[task_id]["test"],
                text=problems[task_id]["text"]
            )
        ]
        write_jsonl("samples.jsonl", samples, True)

2. Pull Humaneval-x Container

docker pull rishubi/codegeex:latest

3. Enter Container and Mount Files

docker run --runtime=nvidia -it -v /home/runner/Firefly/test.py:/work/data/gen_samples.py -v /home/runner/AI-ModelScope/CodeLlama-7b-hf:/work/data/CodeLlama-7b-hf rishubi/codegeex bash

4. Update Container Code

$ cd /workspace/CodeGeeX
$ git pull origin main

5. Modify Evaluation Script (includes five languages, can be manually disalbed or referenced via code)

6. Navigate to Generated File Directory and Install Dependencies

cd /work/data
# Install dependencies
pip install human-eval

7. Upgrade Transformer Library (Docker container has version 4.22 which has bugs; upgrade to 4.28.1)

pip uninstall transformers
pip install transformers==4.28.1

8. Update Tokenizer Library to Version 13.3

pip uninstall tokenizers
pip install tokenizers==0.13.3 -i https://pypi.tuna.tsinghua.edu.cn/simple

9. Execute Generation Script

python gen_samples.py

10. Run Evaluation

bash scripts/evaluate_humaneval_x.sh /work/data/samples.jsonl java
python /workspace/CodeGeeX/codegeex/benchmark/humaneval-x/evaluate_humaneval_x.py

Notes:

1. Replace container's execution.py with the one from GitHub to avoid syntax and indentation errors.
2. Docker environment requires multiple modifications; consider rebuilding when encountering errors during evaluation.

Code Translation Dataset Preparation

Create a gen_json folder in /work/data for storing translation datasets, focusing on adding a tran_prompt field instead of using the original prompt.

from codegeex.benchmark.utils import read_translation_dataset
import json
import gzip

languages = ['cpp', 'java', 'python', 'go', 'rust', 'js']

# Iterate through each language pair
for src_lang in languages:
    for tgt_lang in languages:
        # Skip identical language pairs
        if src_lang == tgt_lang:
            continue

        # Call function to get dataset
        dataset = read_translation_dataset(f'/work/data/CodeGeeX/codegeex/benchmark/humaneval-x/{src_lang}/data/humaneval_{src_lang}.jsonl.gz',
                                           f'/work/data/CodeGeeX/codegeex/benchmark/humaneval-x/{tgt_lang}/data/humaneval_{tgt_lang}.jsonl.gz',
                                           src_lang,
                                           tgt_lang,
                                           'humaneval')

        # Save as JSONL.gz file
        with gzip.open(f'{src_lang}_{tgt_lang}.jsonl.gz', 'wt') as f:
            for task in dataset.values():
                json.dump(task, f)
                f.write('\n')

Then generate translated code with the following script:

# Import required libraries
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
import time
from datetime import datetime
from human_eval.data import write_jsonl, read_problems
from collections import defaultdict
import os
from itertools import permutations

# Configure model and tokenizer paths
print(torch.cuda.is_available())
print(torch.__version__)
model_name = "codellama-7b-jj"
tokenizer_name = "codellama-7b-jj"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
print("Device:", device)
print("Model device:", next(model.parameters()).device)

# Load model and tokenizer
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)

# Set generation parameters
top_p = 0.9
temperature = 0.35
repetition_penalty = 1.0
top_k = 50

import re

# Post-process generated text
def postprocess_generation(generated, prompt, stop_words):
    # Remove prompt
    if generated.startswith(prompt):
        generated = generated[len(prompt):].strip()
    
    # Remove potential test content starting with "#"
    separator_index = generated.find("#")
    if separator_index != -1:
        generated = generated[:separator_index].strip()
    
    # Handle added "}}} " termination marker
    separator_index = generated.find("}}}")
    if separator_index != -1:
        generated = generated[:separator_index].strip()
    
    # Remove generated main functions
    cpp_split_pattern = re.compile(r'\nint main\(\)\s*\{')
    python_split_pattern = re.compile(r'\s*if\s+__name__\s*==\s*["\']__main__["\']\s*:\s*')
    parts1 = cpp_split_pattern.split(generated, maxsplit=1)
    if len(parts1) > 1:
        return parts1[0].strip()
    else:
        parts2 = python_split_pattern.split(parts1[0], maxsplit=1)
        return parts2[0].strip()

# Main generation function
def generate_response(prompt):
    model_input = tokenizer(prompt, return_tensors="pt").to(device)
    model_input.pop("token_type_ids", None)
    model.eval()
    
    with torch.no_grad():
        generated = model.generate(**model_input, max_new_tokens=150, do_sample=True,
                                   temperature=temperature, top_p=top_p, pad_token_id=tokenizer.eos_token_id, repetition_penalty=repetition_penalty)[0]
        generated = tokenizer.decode(generated, skip_special_tokens=True)
    return postprocess_generation(generated, prompt, ["\n\n\n\n\n\n\n\n\n\n"])

def generate_one_completion(task_id, prompt):
    print(datetime.now().strftime("%H:%M:%S"), task_id)
    try:
        result = generate_response(prompt)
        print(result)
        return result
    except Exception as e:
        print(f"Exception occurs: {e}, wait 3 seconds then retry...")
        time.sleep(3)
        generate_one_completion(task_id, prompt)

output_folder = '/work/data/gen_test/'
input_folder = '/work/data/gen_json/'

def process_language_pair(source_lang, target_lang):
    input_file = os.path.join(input_folder, f'{source_lang}_{target_lang}.jsonl.gz')
    output_file = os.path.join(output_folder, f'{source_lang}_{target_lang}_generated.jsonl')
    num_samples_per_task = 1
    problems = read_problems(input_file)
    for task_id in problems:
        samples = []
        for _ in range(num_samples_per_task):
            samples.append({
                'task_id': task_id,
                'generation': generate_one_completion(task_id, problems[task_id]["tran_prompt"]),
                'canonical_solution': problems[task_id].get("canonical_solution", None),
                'declaration': problems[task_id].get("declaration", None),
                'example_test': problems[task_id].get("example_test", None),
                'prompt': problems[task_id].get("prompt", None),
                'test': problems[task_id].get("test", None),
                'text': problems[task_id].get("text", None)
            })
        write_jsonl(output_file, samples, True)

languages = ["python", "java", "cpp", "go", "rust", "js"]
language_pairs = list(permutations(languages, 2))  # Generate all unique pairs

for source_lang, target_lang in language_pairs:
    process_language_pair(source_lang, target_lang)

The generation process completes successfully. The evaluation procedure remains consistant with earlier steps. Key considerations include updating the docker environment and ensuring the correct execution.py file from /opt/python3.8/codegeex is used for evaluation. Minor adjustments may be needed but aligning with the latest GitHub repository ensures proper functionality.