Cirilla

Generall class for creating syntetic datasets with ollama

import os
import random

class Response(BaseModel):
    question: str = Field(description="What question is appropriate to this text?")
    answer: str = Field(description="Answer to the question")

sys_prompt = """
You are an expert dataset annotator for instruction-tuning large language models. 
Your task is to create high-quality question-answer pairs from provided texts 
for training instruct models.

Guidelines:
- Keep the question relevant and informative for learners.
- Avoid using markdown or any unnecessary formatting.
- You can ask to elaborate based on a keyword or phrase in the text.
- You can ask about the plot if the text is a story.
- Do not use overly formal language.
- Use only the information provided in the text.
- If the text states that any part of it is from Netflix, or mentions that a section is from Netflix,
  ignore that part and do not include it in the question or answer.
- If user specifies already created question and answer pair, find a different question and answer
  pair that is different from the one provided. If this is impossible use different words then the ones
  provided.
- Return the output strictly as a JSON with two fields: "question" and "answer".
"""

folder = "./witcher_fandom"
paths = os.listdir(folder)
paths = [os.path.join(folder, p) for p in paths]
print(f"{len(paths)} paths found")

for _ in range(3):
        dual = OllamaCurate(
            model="qwen3:8b",
            system_prompt=sys_prompt,
            response_template=Response
        )
        dual(
            paths,
            save_to="./witcher_synthetic_instruct/qwen3:8b",
            seed=random.randint(0, 1000),
            checkpoint=10
            skip=False,
        )

{
    "question": "What location must the player meet Dijkstra at for the A Deadly Plot quest?",
    "answer": "Passiflora."
}

import os
import random
from pydantic import BaseModel, Field

class Response(BaseModel):
    summary: str = Field(description="Final summary of the entire text. Pure summary only,
    no introduction or reasoning.")

paths = os.listdir("./witcher_fandom")
paths = [os.path.join("./witcher_fandom", p) for p in paths]

print(f"{len(paths)} paths found")

for m in ["granite3.1-moe:3b"]:
    model = OllamaCurate(model=m, system_prompt="", response_template=Response)
    model.single_pass_summary(
        paths,
        save_to=f"./witcher_synth_summaries/{m}",
        seed=random.randint(0, 1000),
        num_predict=4096,
        use_response_template=True,
    )

import os
from pydantic import BaseModel, Field
import os
import random

class Response(BaseModel):
    summary: str = Field(description="Summary of the text, without the thinking process and without any
    introduction. Provide only pure summary, be expressive but stick to the maximum number of words
    that were provided.")

paths = os.listdir('./witcher_texts')
paths = [os.path.join('./witcher_texts', p) for p in paths]

print(f"{len(paths)} paths found")

for m in ['llama3.2:3b', 'llama3.1:8b', 'qwen3:8b']:

    model = OllamaCurate(model=m,
                        system_prompt="",
                        response_template=Response
                        )
    
    model.dynamic_hierarchical_summary(paths,
                                    save_to=f'./synth_sumarries/witcher_texts/{m}',
                                    chunk_lines=100,
                                    seed=random.randint(0, 1000),
                                    num_predict=2048,
                                    max_words_summary=500,
                                    use_response_template=True
                                    )

from pydantic import BaseModel, Field
import os
import random

class Response(BaseModel):
    question: str = Field(description="What question is appropriate to this text?")
    answer: str = Field(description="Answer to the question")

paths = os.listdir('./witcher_fandom')
paths = [os.path.join('./witcher_fandom', p) for p in paths]

bar = tqdm(total=3*3*len(paths))
for model in ['qwen3:8b', 'phi4', 'llama3.1:8b']:

    for _ in range(3):
        ol = OllamaCurate(model, "", Response)
        ol.multi_turn(paths,
        save_to=f'./synth_multi_round/{model}',
        bar=bar,
        n_turns_range=(2, 5),
        seed=random.randint(0, 1000),
        prob_chance_new_context=0.3)

remove duplicate synthetic instructions

Example:

main_dir = './witcher_synthetic_instruct'
save_to = './witcher_synthetic_instruct.jsonl'

rm_duplicate_instructs(main_dir, save_to)

Example input file: ./witcher_synthetic_instruct/qwen3:8b/Deadly Plot.json

{
    "question": "What is the objective of the A Deadly Plot quest in The Witcher 3: Wild Hunt?",
    "answer": "The objective of the A Deadly Plot quest is to help Dijkstra ..."
}

Example input file: ./witcher_synthetic_instruct/qwen3:8b/Deadly Plot_1.json

{
    "question": "What is the main objective of the A Deadly Plot quest in The Witcher 3: Wild Hunt?",
    "answer": "The objective of the A Deadly Plot quest is to help Dijkstra ..."
}

Example created output file (obviously the actual .jsonl file is saved in single lines): ./witcher_synthetic_instruct.jsonl

{"subject": "Silver sword", "text": [
    {"role": "user", "content": "What material are silver swords made from in The Witcher?"},
    {"role": "assistant", "content": "Meteoric iron coated with silver and inscribed with magical runes"}
    ], "data type": "conv", "model": "llama3.1:8b"}
...

turn summaries into a training dataset

Example:

in_path = './witcher_synth_summaries'
out_path = './witcher_summaries_gathered.jsonl'

gather_summaries(in_path, out_path)

Example input file: ./witcher_synth_summaries/llama3.1:8b/Alchemist.txt

In the Witcher lore, Alchemists are individuals who practice alchemy ...

Example created output file (obviously the actual .jsonl file is saved in single lines): ./witcher_summaries_gathered.jsonl

{
    "subject": "sorcerers",
    "text":"Mages, or sorcerers/sorceresses, ...",
    "data type": "plain text", "source": "fandom", "model": "llama3.2:3b"
}
...

turn summaries into simple instructions. The user questions are chosen as one from a list

Example:

in_path = './witcher_synth_summaries'
out_path = './witcher_summaries_gathered_instr.jsonl'

gather_summaries(in_path, out_path)

Example input file: ./witcher_synth_summaries/llama3.1:8b/Alchemist.txt

In the Witcher lore, Alchemists are individuals who practice alchemy ...

Example created output file (obviously the actual .jsonl file is saved in single lines): ./witcher_summaries_gathered_instr.jsonl

{
    "subject": "Ravik",
    "text": [
    {"role": "user", "content": "What is notable about Ravik?"},
    {"role": "assistant", "content": "Ravik, also known as Ravvy, was a friend ..."}
    ],
    "data type": "plain text", "source": "fandom", "model": "llama3.2:3b"
}
...

create synthetic reasoning dataset with reasoning_gym

Example:

out_path = './reason_gym_synth.jsonl'
n_samples = 400 # will contain 40'000 data points

get_synth_resoning_dataset(out_path, n_samples)

Example output file: ./reason_gym_synth.jsonl (obviously the actual .jsonl file is saved in single lines)

{"subject": "simple_equations", "text": [
    {"role": "user", "text": "Solve for g: 65*g = 3185"},
    {"role": "assistant", "text": "49"}],
    "data type": "conv"}
{"subject": "needle_haystack", "text": [
    {"role": "user", "text": "Ismaeel worships lions. Yann embraces travel blogging. Malakhy
    scorns historical documentaries. Kabeer cherishes sketching. \nWho scorns historical
    documentaries? Reply only with a name."},
    {"role": "assistant", "text": "Malakhy"}],
    "data type": "conv"}
...

create synthetic multi turn conversations about given topics with vllm
to see how to turn the instructions into a .jsonl file see the multi_turn_gather function

Example:

paths_ = ['./summaries/granite3.1-moe:3b',
            './summaries/llama3.1:8b',
            './summaries/llama3.2:3b',
            './summaries/qwen3:8b']

paths = [[os.path.join(p, f) for f in os.listdir(p)] for p in paths_]

for model in ["unsloth/granite-3.2-2b-instruct-unsloth-bnb-4bit"]:
    for i, sub_paths in enumerate(paths):
        for _ in range(1):
        
            vllm_multi_turn(sub_paths,
            save_to=f'./synth_multi_round/{model.split("/")[1]}/{paths_[i].split("/")[-1]}',
            model=model)

Where the summaries may come from OllamaCurate.single_pass_summary

Example created output file ./synth_multi_round/granite-3.2-2b-.../granite3.1-moe:3b/A Book of Tales.json:

[
  {
    "question": "Which specific adventure or mystery introduced in The Book of Tales ... ?",
    "answer": "The given text does not specify a single adventure where the ...",
    "context": "A Book of Tales"
  },
  {
    "question": "Which three new playable races \u2013 Gnomes, ... ?",
    "answer": "Gnomes are native to the mountainous region of Mount Carbon in Kovir ...",
    "context": "A Book of Tales"
  },
  {
    "question": "What central role does Radko hold in The Witcher ... ?",
    "answer": "Radko, a soldier under the Bloody Baron's command ...",
    "context": "Radko"
  }
]

gather multi turn conversations into a .jsonl file

Example:

inp = './synth_multi_round'
outp = './multi_round.jsonl'

multi_turn_gather(inp, outp)

Example created output file ./multi_round.jsonl (obviously the actual .jsonl file is saved in single lines):

{
    "subject": "A Little Sacrifice", "text":
    [
    {"role": "user", "content": "What does Sh'eenaz do to help resolve the ... ?"},
    {"role": "assistant", "content": "Sh'eenaz agrees to give up her tail for ..."},
    {"role": "user", "content": "What does Sh'eenaz do to ... ?"},
    {"role": "assistant", "content": "Sh'eenaz ..."},
    {"role": "user", "content": "What is the nature of Geralt's ... ?"},
    {"role": "assistant", "content": "Geralt ..."}
    ],
    "data type": "conv", "source": "fandom",
    "metadata": {"contexts": ["A Little Sacrifice", "A Little Sacrifice", "A Little Sacrifice"]}
}

get an optimized kernel from Huggingface kernel hub
those kernels mostly work only on cuda

Example:

dim=256
activation = get_activation('Motif-Technologies/activation')
rmsnorm = activation.layers.RMSNorm(dim=dim).cuda()

print(rmsnorm(torch.randn(1, dim, device='cuda')).shape)
# torch.Size([1, 256])

BERT attention with grouped query

Signature:

RMSNorm → W_qkv → RoPE → FlashAttention3

Example:

from cirilla.Cirilla_model import benchmark_model_part

att = BertAttention(BertAttentionArgs(), RoPE(128, 512)).cuda().to(torch.bfloat16)
x = torch.randn(4, 512, 128*16, device='cuda', dtype=torch.bfloat16)
benchmark_model_part(att, x, "BertAttention")
# [BertAttention]
# Forward time:   1.85 ms
# Backward time:  4.03 ms
# Forward memory: 56.12 MB
# Backward memory:-36.12 MB

out = att(x)

rotary positional embeddings

Example:

rope = RoPE(128, 512)
xq = torch.randn(2, 512, 4, 128, device='cuda', dtype=torch.bfloat16) # (b, seq_len, head, head_dim)
xk = torch.randn(2, 512, 4, 128, device='cuda', dtype=torch.bfloat16)
xq_out, xk_out = rope.apply_rotary_embeddings(xq, xk)

print(xq.shape, xq_out.shape, xq_out.dtype, xq_out.device)
print(xk.shape, xk_out.shape, xk_out.dtype, xk_out.device)
# torch.Size([2, 512, 4, 128]) torch.Size([2, 512, 4, 128]) torch.bfloat16 cuda:0
# torch.Size([2, 512, 4, 128]) torch.Size([2, 512, 4, 128]) torch.bfloat16 cuda:0

sliding window attention with grouped query

Signature:

RMSNorm → W_qkv → RoPE → FlexAttention

Example:

import torch
from cirilla.LLM_pieces import create_dynamic_block_mask, create_static_block_mask
from attn_gym.mods import generate_tanh_softcap

SOFT_CAP = 20

x = torch.rand((1,2048,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

""" static mask """
static_mask = create_static_block_mask(sliding_window_causal, 2048, 2048)
softcap = generate_tanh_softcap(SOFT_CAP, approx=False)
# or you can do: softcap = None

rope = RoPE(128, 2048)

attention_layer = SlidingWindowAttention(AttentionArgs(),
                                        rope,
                                        mask=static_mask,
                                        score_mod=softcap).to('cuda', dtype=torch.bfloat16)
out = attention_layer(x)
print(out.shape) # torch.Size([1, 2048, 2048])

"""" dynamic mask - won't trigger recompilation """
dynamic_args = AttentionArgs(static_mask=False)
attention_layer = SlidingWindowAttention(dynamic_args,
                                        mask=create_dynamic_block_mask,
                                        rope=rope,
                                        score_mod=softcap).to('cuda', dtype=torch.bfloat16)
out = attention_layer(x)
print(out.shape) # torch.Size([1, 2048, 2048])

x = torch.rand((1,512,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

out = attention_layer(x)
print(out.shape) # torch.Size([1, 512, 2048])


x = torch.rand((1,256,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

out = attention_layer(x)
print(out.shape) # torch.Size([1, 256, 2048])

x = torch.rand((1,2048,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

out = attention_layer(x)
print(out.shape) # torch.Size([1, 2048, 2048])

print(create_dynamic_block_mask.cache_info()) # how many times the mask template was reused
# CacheInfo(hits=1, misses=3, maxsize=32, currsize=3)

a single expert used in Sparse Mixture of Experts (SMoE)

Signature:

SwiGLU (W₁ → [SiLU(x_[:dff]) ⊙ x_[dff:]] → W₂)

pytorch implementation of SMoE

Signature:

RMSNorm → Gating → Experts

Example:

import torch

moe = SMoE(
    SMoEArgs(num_experts=4, k=2),
        [Expert(ExpertArgs()) for _ in range(4)]
    ).to("cuda") # hf kernel only work on cuda

x = torch.randn(4, 1024, 128,
                device='cuda', requires_grad=True) # (b, seq_len, dim) ; requires grad for smoe

out = moe(x)

megablocks implementation of MoE

Example:

import torch
                    
megamoe = MegablockMoE(MegablockArgs())
x = torch.randn(4, 1024, 128,
                device='cuda', requires_grad=True) # (b, seq_len, dim) ; requires grad for smoe

out = megamoe(x)

megablocks implementation of dropless MoE (dMoE)

Example:

import torch

megadmoe = MegablockdMoE(MegablockArgs())
x = torch.randn(4, 1024, 128,
                device='cuda', requires_grad=True) # (b, seq_len, dim) ; requires grad for smoe

out = megadmoe(x)

implementation of ModernBERT with Cirilla LLM pieces

Signature:

Input Embeddings → N×(BertAttention → MoE) → Mean Pooling / tokens / classes / (RMSNorm → W_out)

Example:

model = CirillaBERT(BertArgs(output_what='classify'))

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R', local_checkpoint_folder='./test_model_bert')
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example_bert.jsonl', './example_bert.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer, max_len=model.args.context_window)

from types import MethodType

def new_training_step(self, data):
    out = self.model.pred(data[0], data[1]) # tokens, mask
    loss = self.criterion(out, data[2])
    return loss

trainer.training_step = MethodType(new_training_step, trainer)
trainer.criterion = nn.CrossEntropyLoss()

trainer.train(dl, dl)

vocab_size = 60_000
context_window = 2048
x = torch.randint(0, vocab_size,
    (4, context_window),
    dtype=torch.long)

mask = torch.zeros_like(x)
mask[x != 0] = 1

model = CirillaBERT(BertArgs(output_what='classify', n_classes=2))

print(model.pred(x, mask).shape) # torch.Size([4, 2])

model = CirillaBERT(BertArgs())

mode.pull_model_from_hub('AnthonyPa57/HF-torch-demo-R')

basic dataset for training Cirilla models with .jsonl files

Example:

import time
import numpy as np

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')

dl = JSONLDataset(['./example.jsonl', './example.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer, max_len=32)
print(len(dl))
times = []
dl = DataLoader(dl, batch_size=2)
for _ in range(2):
    times.append(time.time())
    for i in dl:
        print('-'*50)
        print(i[0], i[1], sep='\n')
        times.append(time.time())
print(np.mean(np.diff(times)))

dataset = JSONLDataset(['./example.jsonl', './example.jsonl'],
                        random_missing_char_prob=0.01, random_spelling_mistake_prob=0.02)
for _ in range(4):
    print(dataset._apply_random_spelling_mistake('hello world, I am a sentence'))

# hwllo world, I am a sentence
# hello worls, I am a sentence
# helo world, I am a sentemce
# helloworld, I am a sentence

implementation of modern transformer model with Cirilla LLM pieces

Signature:

Input Embeddings → N×(SlidingWindowAttention → MoE) → RMSNorm → W_out

Example:

model = Cirilla(Args())

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R',
                        local_checkpoint_folder='./test_model')

trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example.jsonl', './example.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer,
                    max_len=model.args.context_window)

trainer.train(dl, dl)

vocab_size = 60_000
context_window = 2048
x = torch.randint(0, vocab_size,
    (4, context_window),
    dtype=torch.long)

model = Cirilla(Args())

print(model.pred(x).shape) # torch.Size([4, context_window, vocab_size])

model = Cirilla(Args())

mode.pull_model_from_hub('AnthonyPa57/HF-torch-demo-R')

benchmark a part of the model

Example:

att = BertAttention(BertAttentionArgs(), RoPE(128, 512)).cuda().to(torch.bfloat16)
x = torch.randn(4, 512, 128*16, device='cuda', dtype=torch.bfloat16)
benchmark_model_part(att, x, "BertAttention")

# [BertAttention]
# Forward time:   1.85 ms
# Backward time:  4.03 ms
# Forward memory: 56.12 MB
# Backward memory:-36.12 MB

tokenizer for the Cirilla models, it retains the functionality of Huggingface tokenizers

Example:

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')

chat = [
    {'role': 'system', 'content': 'What is the capital of France?'},
    {'role': 'user', 'content': 'What is the capital of France?'},
]

print(tokenizer.decode(tokenizer.apply_chat_template(chat, tokenize=True, add_generation_prompt=False)))
print(tokenizer.apply_chat_template(chat, tokenize=True, add_generation_prompt=False))

dl = JSONLDataset('./example.jsonl', shuffle_path=True)

tokenizer = CirillaTokenizer()
tokenizer.train(dl, special_tokens=SPECIAL_TOKENS, min_frequency=2)

tokenizer.push_to_hub('AnthonyPa57/HF-torch-demo2')

print(tokenizer.decode(tokenizer.encode('hello world')))
print(tokenizer.encode('What is the capital of France?'))
print(tokenizer.decode(tokenizer.encode('What is the capital of France?')))

trainer for the Cirilla models

Example:

model = Cirilla(Args())

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R', local_checkpoint_folder='./test_model')
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example.jsonl', './example.jsonl'], shuffle_path=True,
                    tokenizer=tokenizer, max_len=model.args.context_window)

trainer.train(dl, dl)

model = CirillaBERT(BertArgs(output_what='classify'))

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R',
                        local_checkpoint_folder='./test_model_bert')
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example_bert.jsonl', './example_bert.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer, max_len=model.args.context_window)

from types import MethodType

def new_training_step(self, data):
    out = self.model.pred(data[0], data[1]) # tokens, mask
    loss = self.criterion(out, data[2])
    return loss

trainer.training_step = MethodType(new_training_step, trainer)
trainer.criterion = nn.CrossEntropyLoss()

trainer.train(dl, dl)

model = Cirilla(Args())

trainer.benchmark()
# average time for epoch: 0.8215 (seconds)

scrape a given fandom wiki. Use huggingface's span maker (Named Entity Recognition) to search for new pages to scrape

fandom_scraper is a standalone package

Example:

in_path = "./witcher_json"
out_path = "./async_fandom"
instruct_path = "./async_fandom_instruct"

scrape_fandom(in_path,
            out_path,
            instruct_path,
            n_workers = 50,
            wiki = "Witcher",
            lang = "en"
            )

Example input file: ./witcher_json/witcher_1.json

[
    "Geralt of Rivia", "Triss Merigold", "Vesemir", "Leo", "Lambert", 
    "Eskel", "Alvin", "Shani", "Zoltan Chivay", "Dandelion (Jaskier)", 
    "King Foltest", "Adda the White", ...
]

Example created output file: ./async_fandom/Geralt of Rivia.txt

Geralt of Rivia
Sub-Pages:
Main
Biography
Geralt was born as the son of the sorceress Visenna and presumably, the warrior Korin. Shortly after his
birth, his mother left him with the School of the Wolf at the stronghold of Kaer Morhen. There, Geralt
was made and trained to become a Witcher.
As a child, he ...

Example created output file: ./async_fandom_instruct/Geralt of Rivia.json

{
    "Who is Geralt of Rivia?": "Geralt of Rivia is a witcher and the protagonist from The Witcher ...",
    "What is Geralt's nickname?": "Geralt has a number of nicknames, with the more well known ones ...",
    "Who trained Geralt to become a Witcher?": "Geralt was trained by his mentor Vesemir, who he ...",
    "What is Geralt's profession?": "Geralt is a monster slayer for hire. He travels the world on ...",
    "What is the Trial of The Grasses?": "The Trial of The Grasses was a painful process that ..."
}

convert scraped fandom instructions into a .jsonl file

Example:

instructions_into_conv(input_path="./async_fandom_instruct",
                        out_path="./async_fandom_instruct_gathered.jsonl")

Example input file: ./async_fandom_instruct/Geralt of Rivia.json

{
  "Who is Geralt of Rivia?": "Geralt of Rivia is a witcher and the protagonist from The Witcher ...",
  "What is Geralt's nickname?": "Geralt has a number of nicknames, with the more well known ones ...",
  "Who trained Geralt to become a Witcher?": "Geralt was trained by his mentor Vesemir, who he ...",
  "What is Geralt's profession?": "Geralt is a monster slayer for hire. He travels the world on ...",
  "What is the Trial of The Grasses?": "The Trial of The Grasses was a painful process that ..."
}

Example created output file (obviously the actual .jsonl file is saved in single lines): ./async_fandom_instruct_gathered.jsonl

{"subject": "Geralt of Rivia", "text": [
    {"role": "user", "content": "Who is Geralt of Rivia?"},
    {"role": "assistant", "content": "Geralt of Rivia is a witcher and the protagonist from ..."}
    ], "data type": "conv", "source": "fandom"}
{"subject": "Geralt of Rivia", "text": [
    {"role": "user", "content": "What is Geralt's nickname?"},
    {"role": "assistant", "content": "Geralt has a number of nicknames, with the more well known ones ..."}
    ], "data type": "conv", "source": "fandom"}
...