Говорили про антиспам-бота на эмбеддингах.

Вот так он обучается:

import os
from itertools import batched

import joblib
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from transformers import AutoTokenizer, AutoModel

from app.tools import MinioInit
from app.utils import prepare_data, embed

tokenizer = AutoTokenizer.from_pretrained("cointegrated/rubert-tiny2")
model = AutoModel.from_pretrained("cointegrated/rubert-tiny2")


def main():
    spam_path = "./data/spam.json"
    ham_path = "./data/ham.json"
    whitelist_path = "./model/whitelist.json"
    assert os.path.exists(whitelist_path)
    assert os.path.exists(ham_path)
    assert os.path.exists(spam_path)

    spam = prepare_data(spam_path)
    ham = prepare_data(ham_path)

    print(spam.shape, ham.shape)

    preds = []
    n = spam.shape[0] * 11
    train_y = np.hstack([np.zeros(n), np.ones(spam.shape[0])])
    for batch in batched(ham, n):
        if len(batch) == n:
            train_X = np.vstack([np.vstack(batch), spam])
            # clf = RandomForestClassifier(random_state=20250409, n_estimators=100)
            clf = KNeighborsClassifier(n_neighbors=5)
            clf.fit(train_X, train_y)
            preds.append(clf.predict(ham))
    score_ham = np.mean(preds, axis=0)

    print(len(preds))
    print((score_ham > 0).sum(), n)

    train_X = np.vstack([ham[np.argsort(score_ham)[-n:]], spam])
    clf.fit(train_X, train_y)

    joblib.dump(clf, './model/spam_ham_n.pkl')

    mi = MinioInit()

    print("save_file")
    with open('./model/spam_ham_n.pkl', 'rb') as f:
        mi.save_file(f, 'model/spam_ham_n.pkl')

    if os.path.exists('./model/spam_ham_n.pkl'):
        os.remove('./model/spam_ham_n.pkl')

    print("Загружаем модель")
    clf2 = mi.load_data('model/spam_ham_n.pkl')

    test = clf2.predict_proba(embed("""От 300 $ в день""").reshape(1, -1))[0, 1]

    print('test ', test)


if __name__ == "__main__":
    main()