Representation Engineering (RepE)

import torch
from transformers import AutoModel, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModel.from_pretrained("gpt2")
template = "Consider the amount of happiness in the following scenario:\n{}\nThe amount of happiness in the scenario is"


def read(scenario, layer=8):
    """Return the last token's hidden state at one layer for a scenario wrapped in the template."""
    with torch.no_grad():
        outputs = model(**tokenizer(template.format(scenario), return_tensors="pt"), output_hidden_states=True)
    return outputs.hidden_states[layer][0, -1]


# Stimulus pairs that differ only in the concept being read
pairs = [
    ("I passed the exam.", "I failed the exam."),
    ("We won the match.", "We lost the match."),
    ("She found her keys.", "She lost her keys."),
    ("The party was fun.", "The party was dull."),
    ("My plant is thriving.", "My plant is dying."),
    ("He got the job.", "He lost the job."),
]
diffs = torch.stack([read(happy) - read(sad) for happy, sad in pairs])
direction = torch.linalg.svd(diffs, full_matrices=False).Vh[0]  # top principal direction of the differences
direction *= (diffs @ direction).mean().sign()  # orient it toward the happy member of each pair
center = torch.stack([read(text) for pair in pairs for text in pair]).mean(0)

# Score unseen scenarios by projecting their activations onto the reading vector
for scenario in [
    "The surgery went well.",
    "Our flight was canceled.",
    "We adopted a puppy.",
    "My phone was stolen.",
    "I got a raise today.",
    "The basement flooded.",
]:
    print(f"{(read(scenario) - center) @ direction:+6.1f}  {scenario}")