Cirilla

Generall class for creating syntetic datasets with ollama

import os
import random

class Response(BaseModel):
    question: str = Field(description="What question is appropriate to this text?")
    answer: str = Field(description="Answer to the question")

sys_prompt = """
You are an expert dataset annotator for instruction-tuning large language models.
Your task is to create high-quality question-answer pairs from provided texts
for training instruct models.

Guidelines:
- Keep the question relevant and informative for learners.
- Avoid using markdown or any unnecessary formatting.
- You can ask to elaborate based on a keyword or phrase in the text.
- You can ask about the plot if the text is a story.
- Do not use overly formal language.
- Use only the information provided in the text.
- If the text states that any part of it is from Netflix, or mentions that a section is from Netflix,
  ignore that part and do not include it in the question or answer.
- If user specifies already created question and answer pair, find a different question and answer
  pair that is different from the one provided. If this is impossible use different words then the ones
  provided.
- Return the output strictly as a JSON with two fields: "question" and "answer".
"""

folder = "./witcher_fandom"
paths = os.listdir(folder)
paths = [os.path.join(folder, p) for p in paths]
print(f"{len(paths)} paths found")

for _ in range(3):
        dual = OllamaCurate(
            model="qwen3:8b",
            system_prompt=sys_prompt,
            response_template=Response
        )
        dual(
            paths,
            save_to="./witcher_synthetic_instruct/qwen3:8b",
            seed=random.randint(0, 1000),
            checkpoint=10
            skip=False,
        )

{
    "question": "What location must the player meet Dijkstra at for the A Deadly Plot quest?",
    "answer": "Passiflora."
}

import os
import random
from pydantic import BaseModel, Field

class Response(BaseModel):
    summary: str = Field(description="Final summary of the entire text. Pure summary only,
    no introduction or reasoning.")

paths = os.listdir("./witcher_fandom")
paths = [os.path.join("./witcher_fandom", p) for p in paths]

print(f"{len(paths)} paths found")

for m in ["granite3.1-moe:3b"]:
    model = OllamaCurate(model=m, system_prompt="", response_template=Response)
    model.single_pass_summary(
        paths,
        save_to=f"./witcher_synth_summaries/{m}",
        seed=random.randint(0, 1000),
        num_predict=4096,
        use_response_template=True,
    )

import os
from pydantic import BaseModel, Field
import os
import random

class Response(BaseModel):
    summary: str = Field(description="Summary of the text, without the thinking process and without any
    introduction. Provide only pure summary, be expressive but stick to the maximum number of words
    that were provided.")

paths = os.listdir('./witcher_texts')
paths = [os.path.join('./witcher_texts', p) for p in paths]

print(f"{len(paths)} paths found")

for m in ['llama3.2:3b', 'llama3.1:8b', 'qwen3:8b']:

    model = OllamaCurate(model=m,
                        system_prompt="",
                        response_template=Response
                        )

    model.dynamic_hierarchical_summary(paths,
                                    save_to=f'./synth_sumarries/witcher_texts/{m}',
                                    chunk_lines=100,
                                    seed=random.randint(0, 1000),
                                    num_predict=2048,
                                    max_words_summary=500,
                                    use_response_template=True
                                    )

from pydantic import BaseModel, Field
import os
import random

class Response(BaseModel):
    question: str = Field(description="What question is appropriate to this text?")
    answer: str = Field(description="Answer to the question")

paths = os.listdir('./witcher_fandom')
paths = [os.path.join('./witcher_fandom', p) for p in paths]

bar = tqdm(total=3*3*len(paths))
for model in ['qwen3:8b', 'phi4', 'llama3.1:8b']:

    for _ in range(3):
        ol = OllamaCurate(model, "", Response)
        ol.multi_turn(paths,
        save_to=f'./synth_multi_round/{model}',
        bar=bar,
        n_turns_range=(2, 5),
        seed=random.randint(0, 1000),
        prob_chance_new_context=0.3)

remove duplicate synthetic instructions

Example:

main_dir = './witcher_synthetic_instruct'
save_to = './witcher_synthetic_instruct.jsonl'

rm_duplicate_instructs(main_dir, save_to)

Example input file: ./witcher_synthetic_instruct/qwen3:8b/Deadly Plot.json

{
    "question": "What is the objective of the A Deadly Plot quest in The Witcher 3: Wild Hunt?",
    "answer": "The objective of the A Deadly Plot quest is to help Dijkstra ..."
}

Example input file: ./witcher_synthetic_instruct/qwen3:8b/Deadly Plot_1.json

{
    "question": "What is the main objective of the A Deadly Plot quest in The Witcher 3: Wild Hunt?",
    "answer": "The objective of the A Deadly Plot quest is to help Dijkstra ..."
}

Example created output file (obviously the actual .jsonl file is saved in single lines): ./witcher_synthetic_instruct.jsonl

{"subject": "Silver sword", "text": [
    {"role": "user", "content": "What material are silver swords made from in The Witcher?"},
    {"role": "assistant", "content": "Meteoric iron coated with silver and inscribed with magical runes"}
    ], "data type": "conv", "model": "llama3.1:8b"}
...

turn summaries into a training dataset

Example:

in_path = './witcher_synth_summaries'
out_path = './witcher_summaries_gathered.jsonl'

gather_summaries(in_path, out_path)

Example input file: ./witcher_synth_summaries/llama3.1:8b/Alchemist.txt

In the Witcher lore, Alchemists are individuals who practice alchemy ...

Example created output file (obviously the actual .jsonl file is saved in single lines): ./witcher_summaries_gathered.jsonl

{
    "subject": "sorcerers",
    "text":"Mages, or sorcerers/sorceresses, ...",
    "data type": "plain text", "source": "fandom", "model": "llama3.2:3b"
}
...

turn summaries into simple instructions. The user questions are chosen as one from a list

Example:

in_path = './witcher_synth_summaries'
out_path = './witcher_summaries_gathered_instr.jsonl'

gather_summaries(in_path, out_path)

Example input file: ./witcher_synth_summaries/llama3.1:8b/Alchemist.txt

In the Witcher lore, Alchemists are individuals who practice alchemy ...

Example created output file (obviously the actual .jsonl file is saved in single lines): ./witcher_summaries_gathered_instr.jsonl

{
    "subject": "Ravik",
    "text": [
    {"role": "user", "content": "What is notable about Ravik?"},
    {"role": "assistant", "content": "Ravik, also known as Ravvy, was a friend ..."}
    ],
    "data type": "plain text", "source": "fandom", "model": "llama3.2:3b"
}
...

create synthetic reasoning dataset with reasoning_gym

Example:

out_path = './reason_gym_synth.jsonl'
n_samples = 400 # will contain 40'000 data points

get_synth_reasoning_dataset(out_path, n_samples)

Example output file: ./reason_gym_synth.jsonl (obviously the actual .jsonl file is saved in single lines)

{"subject": "simple_equations", "text": [
    {"role": "user", "text": "Solve for g: 65*g = 3185"},
    {"role": "assistant", "text": "49"}],
    "data type": "conv"}
{"subject": "needle_haystack", "text": [
    {"role": "user", "text": "Ismaeel worships lions. Yann embraces travel blogging. Malakhy
    scorns historical documentaries. Kabeer cherishes sketching. \nWho scorns historical
    documentaries? Reply only with a name."},
    {"role": "assistant", "text": "Malakhy"}],
    "data type": "conv"}
...

create synthetic multi turn conversations about given topics with vllm
to see how to turn the instructions into a .jsonl file see the multi_turn_gather function

Example:

paths_ = ['./summaries/granite3.1-moe:3b',
            './summaries/llama3.1:8b',
            './summaries/llama3.2:3b',
            './summaries/qwen3:8b']

paths = [[os.path.join(p, f) for f in os.listdir(p)] for p in paths_]

for model in ["unsloth/granite-3.2-2b-instruct-unsloth-bnb-4bit"]:
    for i, sub_paths in enumerate(paths):
        for _ in range(1):

            vllm_multi_turn(sub_paths,
            save_to=f'./synth_multi_round/{model.split("/")[1]}/{paths_[i].split("/")[-1]}',
            model=model)

Where the summaries may come from OllamaCurate.single_pass_summary

Example created output file ./synth_multi_round/granite-3.2-2b-.../granite3.1-moe:3b/A Book of Tales.json:

[
  {
    "question": "Which specific adventure or mystery introduced in The Book of Tales ... ?",
    "answer": "The given text does not specify a single adventure where the ...",
    "context": "A Book of Tales"
  },
  {
    "question": "Which three new playable races – Gnomes, ... ?",
    "answer": "Gnomes are native to the mountainous region of Mount Carbon in Kovir ...",
    "context": "A Book of Tales"
  },
  {
    "question": "What central role does Radko hold in The Witcher ... ?",
    "answer": "Radko, a soldier under the Bloody Baron's command ...",
    "context": "Radko"
  }
]

gather multi turn conversations into a .jsonl file

Example:

inp = './synth_multi_round'
outp = './multi_round.jsonl'

multi_turn_gather(inp, outp)

Example created output file ./multi_round.jsonl (obviously the actual .jsonl file is saved in single lines):

{
    "subject": "A Little Sacrifice", "text":
    [
    {"role": "user", "content": "What does Sh'eenaz do to help resolve the ... ?"},
    {"role": "assistant", "content": "Sh'eenaz agrees to give up her tail for ..."},
    {"role": "user", "content": "What does Sh'eenaz do to ... ?"},
    {"role": "assistant", "content": "Sh'eenaz ..."},
    {"role": "user", "content": "What is the nature of Geralt's ... ?"},
    {"role": "assistant", "content": "Geralt ..."}
    ],
    "data type": "conv", "source": "fandom",
    "metadata": {"contexts": ["A Little Sacrifice", "A Little Sacrifice", "A Little Sacrifice"]}
}

get an optimized kernel from Huggingface kernel hub
those kernels mostly work only on cuda

Example:

dim=256
activation = get_activation('Motif-Technologies/activation')
rmsnorm = activation.layers.RMSNorm(dim=dim).cuda()

print(rmsnorm(torch.randn(1, dim, device='cuda')).shape)
# torch.Size([1, 256])

BERT attention with grouped query

Signature:

RMSNorm → W_qkv → RoPE → FlexAttention

Example:

from cirilla.Cirilla_model import benchmark_model_part

att = BertAttention(BertAttentionArgs(), RoPE(128, 512)).cuda().to(torch.bfloat16)
x = torch.randn(4, 512, 128*16, device='cuda', dtype=torch.bfloat16)
benchmark_model_part(att, x, "BertAttention")
# [BertAttention]
# Forward time:   1.85 ms
# Backward time:  4.03 ms
# Forward memory: 56.12 MB
# Backward memory:-36.12 MB

out = att(x)

rotary positional embeddings

Example:

rope = RoPE(128, 512)
xq = torch.randn(2, 512, 4, 128, device='cuda', dtype=torch.bfloat16) # (b, seq_len, head, head_dim)
xk = torch.randn(2, 512, 4, 128, device='cuda', dtype=torch.bfloat16)
xq_out, xk_out = rope.apply_rotary_embeddings(xq, xk)

print(xq.shape, xq_out.shape, xq_out.dtype, xq_out.device)
print(xk.shape, xk_out.shape, xk_out.dtype, xk_out.device)
# torch.Size([2, 512, 4, 128]) torch.Size([2, 512, 4, 128]) torch.bfloat16 cuda:0
# torch.Size([2, 512, 4, 128]) torch.Size([2, 512, 4, 128]) torch.bfloat16 cuda:0

sliding window attention with grouped query

Signature:

RMSNorm → W_qkv → RoPE → FlexAttention

Example:

import torch
from cirilla.LLM_pieces import create_dynamic_block_mask, create_static_block_mask
from attn_gym.mods import generate_tanh_softcap

SOFT_CAP = 20

x = torch.rand((1,2048,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

""" static mask """
static_mask = create_static_block_mask(sliding_window_causal, 2048, 2048)
softcap = generate_tanh_softcap(SOFT_CAP, approx=False)
# or you can do: softcap = None

rope = RoPE(128, 2048)

attention_layer = SlidingWindowAttention(AttentionArgs(),
                                        rope,
                                        mask=static_mask,
                                        score_mod=softcap).to('cuda', dtype=torch.bfloat16)
out = attention_layer(x)
print(out.shape) # torch.Size([1, 2048, 2048])

"""" dynamic mask - won't trigger recompilation """
dynamic_args = AttentionArgs(static_mask=False)
attention_layer = SlidingWindowAttention(dynamic_args,
                                        mask=create_dynamic_block_mask,
                                        rope=rope,
                                        score_mod=softcap).to('cuda', dtype=torch.bfloat16)
out = attention_layer(x)
print(out.shape) # torch.Size([1, 2048, 2048])

x = torch.rand((1,512,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

out = attention_layer(x)
print(out.shape) # torch.Size([1, 512, 2048])


x = torch.rand((1,256,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

out = attention_layer(x)
print(out.shape) # torch.Size([1, 256, 2048])

x = torch.rand((1,2048,128*16), device='cuda', dtype=torch.bfloat16) # (b, seq, head_dim*h)

out = attention_layer(x)
print(out.shape) # torch.Size([1, 2048, 2048])

print(create_dynamic_block_mask.cache_info()) # how many times the mask template was reused
# CacheInfo(hits=1, misses=3, maxsize=32, currsize=3)

Standalone SwiGLU feed-forward block used as the expert FFN inside SMoE. Can also be used independently as a feed-forward layer.

Signature:

(W_1a(x) ⊙ W_1b(x)) → SiLU → W₂

Example:

from cirilla.LLM_pieces import SwiGLU
from cirilla.LLM_pieces.SMoE import SwiGLUArgs
import torch

ffn = SwiGLU(SwiGLUArgs(dim=256, d_ff=512)).cuda().to(torch.bfloat16)
x = torch.randn(4, 128, 256, device='cuda', dtype=torch.bfloat16)
print(ffn(x).shape)  # torch.Size([4, 128, 256])

pytorch implementation of SMoE

Signature:

RMSNorm → Gating → Experts

Example:

import torch

moe = SMoE(
    SMoEArgs(num_experts=4, k=2),
        [SwiGLU(SwiGLUArgs()) for _ in range(4)]
    ).to("cuda") # hf kernel only work on cuda

x = torch.randn(4, 1024, 128,
                device='cuda', requires_grad=True) # (b, seq_len, dim) ; requires grad for smoe

out = moe(x)

Normalization-free alternative to RMSNorm from the Derf paper. Uses a learnable erf activation as the normalizer: weight · erf(α·x + shift) + bias. Use as a drop-in replacement by passing layer_norm='Derf' in any model Args dataclass.

Example:

from cirilla.LLM_pieces import Dynamic_erf
import torch

norm = Dynamic_erf(256).cuda().to(torch.bfloat16)
x = torch.randn(4, 128, 256, device='cuda', dtype=torch.bfloat16)
print(norm(x).shape)  # torch.Size([4, 128, 256])

# Enable in any model via Args:
model = Cirilla(Args(layer_norm='Derf'))

Normalization-free alternative to RMSNorm from the Derf paper. Uses a learnable tanh activation: weight · tanh(α·x) + bias. Enable with layer_norm='DyT' in any model Args dataclass.

Example:

from cirilla.LLM_pieces import DynamicTanh
import torch

norm = DynamicTanh(256).cuda().to(torch.bfloat16)
x = torch.randn(4, 128, 256, device='cuda', dtype=torch.bfloat16)
print(norm(x).shape)  # torch.Size([4, 128, 256])

# Enable in any model via Args:
model = Cirilla(Args(layer_norm='DyT'))

implementation of ModernBERT with Cirilla LLM pieces

Signature:

Input Embeddings → N×(BertAttention → MoE) → Mean Pooling / tokens / classes / (RMSNorm → W_out)

Example:

model = CirillaBERT(BertArgs(output_what='classify'))

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R', local_checkpoint_folder='./test_model_bert')
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example_bert.jsonl', './example_bert.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer, max_len=model.args.context_window)

from types import MethodType

def new_training_step(self, data):
    out = self.model.pred(data[0], data[1]) # tokens, mask
    loss = self.criterion(out, data[2])
    return loss

trainer.training_step = MethodType(new_training_step, trainer)
trainer.criterion = nn.CrossEntropyLoss()

trainer.train(dl, dl)

vocab_size = 60_000
context_window = 2048
x = torch.randint(0, vocab_size,
    (4, context_window),
    dtype=torch.long)

mask = torch.zeros_like(x)
mask[x != 0] = 1

model = CirillaBERT(BertArgs(output_what='classify', n_classes=2))

print(model.pred(x, mask).shape) # torch.Size([4, 2])

model = CirillaBERT(BertArgs())

mode.pull_model_from_hub('AnthonyPa57/HF-torch-demo-R')

basic dataset for training Cirilla models with .jsonl files

Example:

import time
import numpy as np

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')

dl = JSONLDataset(['./example.jsonl', './example.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer, max_len=32)
print(len(dl))
times = []
dl = DataLoader(dl, batch_size=2)
for _ in range(2):
    times.append(time.time())
    for i in dl:
        print('-'*50)
        print(i[0], i[1], sep='\n')
        times.append(time.time())
print(np.mean(np.diff(times)))

dataset = JSONLDataset(['./example.jsonl', './example.jsonl'],
                        random_missing_char_prob=0.01, random_spelling_mistake_prob=0.02)
for _ in range(4):
    print(dataset._apply_random_spelling_mistake('hello world, I am a sentence'))

# hwllo world, I am a sentence
# hello worls, I am a sentence
# helo world, I am a sentemce
# helloworld, I am a sentence

implementation of modern transformer model with Cirilla LLM pieces

Signature:

Input Embeddings → N×(SlidingWindowAttention → MoE) → RMSNorm → W_out

Example:

model = Cirilla(Args())

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R',
                        local_checkpoint_folder='./test_model')

trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example.jsonl', './example.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer,
                    max_len=model.args.context_window)

trainer.train(dl, dl)

vocab_size = 60_000
context_window = 2048
x = torch.randint(0, vocab_size,
    (4, context_window),
    dtype=torch.long)

model = Cirilla(Args())

print(model.pred(x).shape) # torch.Size([4, context_window, vocab_size])

model = Cirilla(Args())

mode.pull_model_from_hub('AnthonyPa57/HF-torch-demo-R')

benchmark a part of the model

Example:

att = BertAttention(BertAttentionArgs(), RoPE(128, 512)).cuda().to(torch.bfloat16)
x = torch.randn(4, 512, 128*16, device='cuda', dtype=torch.bfloat16)
benchmark_model_part(att, x, "BertAttention")

# [BertAttention]
# Forward time:   1.85 ms
# Backward time:  4.03 ms
# Forward memory: 56.12 MB
# Backward memory:-36.12 MB

tokenizer for the Cirilla models, it retains the functionality of Huggingface tokenizers

Example:

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')

chat = [
    {'role': 'system', 'content': 'What is the capital of France?'},
    {'role': 'user', 'content': 'What is the capital of France?'},
]

print(tokenizer.decode(tokenizer.apply_chat_template(chat, tokenize=True, add_generation_prompt=False)))
print(tokenizer.apply_chat_template(chat, tokenize=True, add_generation_prompt=False))

dl = JSONLDataset('./example.jsonl', shuffle_path=True)

tokenizer = CirillaTokenizer()
tokenizer.train(dl, special_tokens=SPECIAL_TOKENS, min_frequency=2)

tokenizer.push_to_hub('AnthonyPa57/HF-torch-demo2')

print(tokenizer.decode(tokenizer.encode('hello world')))
print(tokenizer.encode('What is the capital of France?'))
print(tokenizer.decode(tokenizer.encode('What is the capital of France?')))

trainer for the Cirilla models

Example:

model = Cirilla(Args())

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R', local_checkpoint_folder='./test_model')
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example.jsonl', './example.jsonl'], shuffle_path=True,
                    tokenizer=tokenizer, max_len=model.args.context_window)

trainer.train(dl, dl)

model = CirillaBERT(BertArgs(output_what='classify'))

targs = TrainingArgs(hf_repo_id='AnthonyPa57/HF-torch-demo-R',
                        local_checkpoint_folder='./test_model_bert')
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example_bert.jsonl', './example_bert.jsonl'],
                    shuffle_path=True, tokenizer=tokenizer, max_len=model.args.context_window)

from types import MethodType

def new_training_step(self, data):
    out = self.model.pred(data[0], data[1]) # tokens, mask
    loss = self.criterion(out, data[2])
    return loss

trainer.training_step = MethodType(new_training_step, trainer)
trainer.criterion = nn.CrossEntropyLoss()

trainer.train(dl, dl)

model = Cirilla(Args())

trainer.benchmark()
# average time for epoch: 0.8215 (seconds)

Multi-Token Prediction model. Extends Cirilla with n_token_heads additional lightweight single-layer decoder heads, each predicting the next k-th token ahead in parallel. This improves convergence speed and sample efficiency with minimal extra compute.

Signature:

Input Embeddings → N×(SlidingWindowAttention → MoE) → RMSNorm → W_out
     + n×(Decoder → RMSNorm → W_out)  [one head per future token]

Example:

from cirilla.Cirilla_model import (
    CirillaMTP, MTPArgs, CirillaTrainer, TrainingArgs,
    CirillaTokenizer, JSONLDataset, mtp_training_step, mtp_inference_step
)
from types import MethodType
from functools import partial

model = CirillaMTP(MTPArgs(n_layers=4, dim=128, d_ff=256,
                            n_heads=8, context_window=128,
                            n_token_heads=4))

targs = TrainingArgs(n_epoch=1000)
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
pad_id = tokenizer.tokenizer.pad_token_id

dl = JSONLDataset(['./example.jsonl'], tokenizer=tokenizer,
                    max_len=model.args.context_window)

trainer.training_step = MethodType(partial(mtp_training_step, pad_id=pad_id), trainer)
trainer.inference_step = MethodType(partial(mtp_inference_step, pad_id=pad_id), trainer)
trainer.criterion = None

trainer.train(dl, dl)

Tiny Recursive Model. Wraps any nn.Module in an adaptive computation loop: the same network refines an answer vector y_hat and latent state z across multiple steps. A learnable halt gate decides when to stop early at inference time — shallow inputs get fewer steps, harder inputs get more.

Example:

from cirilla.Cirilla_model import (
    CirillaTRM, TRMArgs, MLPMixer1D, MixerArgs,
    CirillaTrainer, TrainingArgs, CirillaTokenizer, JSONLDataset,
    trm_training_step, trm_inference_step
)
from ema_pytorch import EMA
from types import MethodType
from functools import partial

mixer = MLPMixer1D(MixerArgs())
model = CirillaTRM(mixer, TRMArgs())

ema_model = EMA(model, beta=0.999,
                update_model_with_ema_every=1_000,
                forward_method_names=('predict',))

targs = TrainingArgs(n_epoch=1000)
trainer = CirillaTrainer(model, targs)

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
dl = JSONLDataset(['./example.jsonl'], tokenizer=tokenizer,
                    max_len=mixer.args.context_window)

trainer.training_step = MethodType(
    partial(trm_training_step, max_recurrent_step=16,
            halt_weight=0.5, halt_thresh=0.5, ema_model=ema_model), trainer)
trainer.inference_step = MethodType(
    partial(trm_inference_step, max_recurrent_step=16, halt_thresh=0.5), trainer)
trainer.criterion = None

trainer.train(dl, dl)

Vision-Language model. A Swin Transformer encodes image patches into token embeddings which are prepended to the text token sequence and passed into a Cirilla decoder — similar architecture to PaliGemma.

Signature:

Image → Swin Transformer → image tokens → concat(image tokens, text tokens) → N×(SlidingWindowAttention → MoE) → RMSNorm → W_out

Transformer–Mamba hybrid model. Replaces the standard transformer decoder with HybridDecoder blocks that interleave Mamba SSM layers with transformer attention layers (similar to IBM Granite 4.0). Requires the optional mamba dependency.

Example:

# requires: uv add Cirilla[mamba]
from cirilla.Cirilla_model import HybridCirilla, HybridArgs

model = HybridCirilla(HybridArgs(n_layers=5, layer_pattern='5M'))
print(model.n_params / 1e6, "M")

Configure optimizers for a Cirilla model. Optionally combines a standard optimizer (default AdamW) for embedding and output layers with the Muon optimizer for hidden layers, which can improve convergence on transformer hidden weights.

Example:

from cirilla.Cirilla_model import Cirilla, Args, get_optims

model = Cirilla(Args())
optims = get_optims(model, use_muon_optim=True, lr=3e-4, weight_decay=0.1)

Auxiliary load-balancing loss for MoE models. Encourages uniform expert utilisation by penalising skewed routing distributions. Add a small multiple (e.g. 0.01) of this to the main training loss. Requires output_moe_weights=True in the model args.

Example:

from cirilla.Cirilla_model import Cirilla, Args, load_balancing_loss
import torch.nn as nn

model = Cirilla(Args(output_moe_weights=True))
criterion = nn.CrossEntropyLoss()

out, moe_weights = model.pred(x)
lb_losses = [
    load_balancing_loss(w,
    num_experts=model.args.num_experts,
    top_k=model.args.k)
    for w in moe_weights
        ]
lb_loss = torch.stack(lb_losses).mean()
loss = criterion(out.view(-1, out.shape[-1]),
    targets.view(-1))
loss = loss + 0.01 * lb_loss
loss.backward()

Non-causal (bidirectional) encoder block: N layers of BertAttention interleaved with SMoE. Used as the recursive core in CirillaTRM and as a standalone encoder for classification / embedding tasks.

Signature:

N×(BertAttention → SMoE)

Example:

from cirilla.Cirilla_model import Encoder, EncoderArgs
import torch

enc = Encoder(EncoderArgs(dim=256, n_layers=4, context_window=512)).cuda()
x = torch.randint(0, 60_000, (2, 512))
emb = torch.randn(2, 512, 256, device='cuda', dtype=torch.bfloat16)
out = enc.pred(emb)
print(out.shape)  # torch.Size([2, 512, 256])

Causal decoder block: N layers of SlidingWindowAttention interleaved with SMoE. The building block inside Cirilla, CirillaMTP, and the token heads of MTP.

Signature:

N×(SlidingWindowAttention → SMoE)

Example:

from cirilla.Cirilla_model import Decoder, DecoderArgs
import torch

dec = Decoder(DecoderArgs(dim=256, n_layers=4, context_window=512,
                            torch_compile=False)).cuda()
x = torch.randn(2, 512, 256, device='cuda', dtype=torch.bfloat16)
out = dec.pred(x)
print(out.shape)  # torch.Size([2, 512, 256])

1-D MLP-Mixer: alternating token-mixing (Conv1d across sequence) and channel-mixing (Linear across features) blocks with pre-norm residual connections. Commonly used as the recursive core network inside CirillaTRM.

Example:

from cirilla.Cirilla_model import MLPMixer1D, MixerArgs, CirillaTRM, TRMArgs
import torch

mixer = MLPMixer1D(MixerArgs(dim=256, depth=6, context_window=512))
model = CirillaTRM(mixer, TRMArgs(dim=256))
print(model.n_params / 1e6, "M")

Variant of JSONLDataset that yields variable-length tensors (no padding). Use together with DynamicCollator as the DataLoader collate function to pad each batch to its longest sequence rather than padding to a global max_len.

Accepts the same arguments as JSONLDataset.

Example:

from cirilla.Cirilla_model import (
    JSONDynamicDatset, DynamicCollator, CirillaTokenizer
)
from torch.utils.data import DataLoader

tokenizer = CirillaTokenizer(hub_url='AnthonyPa57/HF-torch-demo2')
pad_id = tokenizer.tokenizer.pad_token_id

ds = JSONDynamicDatset(['./example.jsonl'], tokenizer=tokenizer, max_len=512)
dl = DataLoader(ds, batch_size=8, collate_fn=DynamicCollator(pad_id))

Collate function for use with JSONDynamicDatset. Pads each batch to the length of its longest sequence using pad_token_id, rather than padding all sequences to a global maximum.

Training step for CirillaMTP. Computes cross-entropy loss across all n_token_heads prediction heads. Bind to CirillaTrainer via MethodType.

Example:

from types import MethodType
from functools import partial
from cirilla.Cirilla_model import mtp_training_step, mtp_inference_step

trainer.training_step = MethodType(partial(mtp_training_step, pad_id=pad_id), trainer)
trainer.inference_step = MethodType(partial(mtp_inference_step, pad_id=pad_id), trainer)
trainer.criterion = None

Validation/inference step for CirillaMTP. Same signature as mtp_training_step — computes loss using only the first token head (the main next-token prediction head). Bind to CirillaTrainer via MethodType.

Training step for CirillaTRM. Unrolls the recursive refinement loop, computes classification cross-entropy loss, and adds a weighted halt loss that encourages the model to stop as early as possible.

Example:

from types import MethodType
from functools import partial
from cirilla.Cirilla_model import trm_training_step, trm_inference_step

trainer.training_step = MethodType(
    partial(trm_training_step, max_recurrent_step=16,
            halt_weight=0.5, halt_thresh=0.5, ema_model=ema_model), trainer)
trainer.inference_step = MethodType(
    partial(trm_inference_step, max_recurrent_step=16, halt_thresh=0.5), trainer)
trainer.criterion = None

Validation/inference step for CirillaTRM. Runs the adaptive halt loop and reports accuracy. Bind to CirillaTrainer via MethodType.

Default training step for CirillaBERT classification tasks. Calls model.pred(tokens, mask) and computes cross-entropy against the provided labels. Bind to CirillaTrainer via MethodType to override the default LM training step.

Example:

from types import MethodType
from cirilla.Cirilla_model import bert_training_step, bert_inference_step
import torch.nn as nn

trainer.training_step = MethodType(bert_training_step, trainer)
trainer.inference_step = MethodType(bert_inference_step, trainer)
trainer.criterion = nn.CrossEntropyLoss()

Validation/inference step for CirillaBERT classification tasks. Reports accuracy over the batch. Same signature as bert_training_step.

scrape a given fandom wiki. Use huggingface's span maker (Named Entity Recognition) to search for new pages to scrape

fandom_scraper is a standalone package

Example:

in_path = "./witcher_json"
out_path = "./async_fandom"
instruct_path = "./async_fandom_instruct"

scrape_fandom(in_path,
            out_path,
            instruct_path,
            n_workers = 50,
            wiki = "Witcher",
            lang = "en"
            )

Example input file: ./witcher_json/witcher_1.json

[
    "Geralt of Rivia", "Triss Merigold", "Vesemir", "Leo", "Lambert",
    "Eskel", "Alvin", "Shani", "Zoltan Chivay", "Dandelion (Jaskier)",
    "King Foltest", "Adda the White", ...
]

Example created output file: ./async_fandom/Geralt of Rivia.txt

Geralt of Rivia
Sub-Pages:
Main
Biography
Geralt was born as the son of the sorceress Visenna and presumably, the warrior Korin. Shortly after his
birth, his mother left him with the School of the Wolf at the stronghold of Kaer Morhen. There, Geralt
was made and trained to become a Witcher.
As a child, he ...

Example created output file: ./async_fandom_instruct/Geralt of Rivia.json

{
    "Who is Geralt of Rivia?": "Geralt of Rivia is a witcher and the protagonist from The Witcher ...",
    "What is Geralt's nickname?": "Geralt has a number of nicknames, with the more well known ones ...",
    "Who trained Geralt to become a Witcher?": "Geralt was trained by his mentor Vesemir, who he ...",
    "What is Geralt's profession?": "Geralt is a monster slayer for hire. He travels the world on ...",
    "What is the Trial of The Grasses?": "The Trial of The Grasses was a painful process that ..."
}

convert scraped fandom instructions into a .jsonl file

Example:

instructions_into_conv(input_path="./async_fandom_instruct",
                        out_path="./async_fandom_instruct_gathered.jsonl")

Example input file: ./async_fandom_instruct/Geralt of Rivia.json

{
  "Who is Geralt of Rivia?": "Geralt of Rivia is a witcher and the protagonist from The Witcher ...",
  "What is Geralt's nickname?": "Geralt has a number of nicknames, with the more well known ones ...",
  "Who trained Geralt to become a Witcher?": "Geralt was trained by his mentor Vesemir, who he ...",
  "What is Geralt's profession?": "Geralt is a monster slayer for hire. He travels the world on ...",
  "What is the Trial of The Grasses?": "The Trial of The Grasses was a painful process that ..."
}

Example created output file (obviously the actual .jsonl file is saved in single lines): ./async_fandom_instruct_gathered.jsonl

{"subject": "Geralt of Rivia", "text": [
    {"role": "user", "content": "Who is Geralt of Rivia?"},
    {"role": "assistant", "content": "Geralt of Rivia is a witcher and the protagonist from ..."}
    ], "data type": "conv", "source": "fandom"}
{"subject": "Geralt of Rivia", "text": [
    {"role": "user", "content": "What is Geralt's nickname?"},
    {"role": "assistant", "content": "Geralt has a number of nicknames, with the more well known ones ..."}
    ], "data type": "conv", "source": "fandom"}
...

Terminal chat UI. After installing Cirilla, launch a full TUI to chat with any Cirilla model pulled from HuggingFace Hub — no extra setup required.

uv run python -m cirilla.cli

On launch, choose a featured model or type any HuggingFace repo id (e.g. AnthonyPa57/Cirilla-0.3B-4E-grpo). Adjust generation settings live during chat with /set key=value:

Featured models:

• Cirilla-0.3B-4E — base pretrained model
• Cirilla-0.3B-4E-grpo — GRPO fine-tuned
• Cirilla-0.3B-4E-grpo-icl — GRPO + in-context learning fine-tuned