🍳 @databreakfast
Showing posts older than #1385 · Back to latest
import json
from telegram import Update
from telegram.ext import (ApplicationBuilder, ContextTypes, MessageHandler,
filters)
from app.env_conf import SETTINGS
from app.tools import logger, MinioInit, init_logging
from app.utils import embed
init_logging()
THRESHOLD = 0.5
logger.info('Загружаем модель из s3')
mi = MinioInit()
clf = mi.load_data('model/spam_ham_n.pkl')
with open("model/whitelist.json", "r") as fp:
whitelist = json.load(fp)
whitelist = set(whitelist)
def score(txt):
return clf.predict_proba(embed(txt.strip()).reshape(1, -1))[0, 1]
logger.info("Грузим модель")
s = score("От 300 $ в день")
logger.info('score: %s', s)
async def handle_message(update: Update, context: ContextTypes.DEFAULT_TYPE):
if update.message:
message_text = update.message.text
user_id = update.message.from_user.id
chat_id = update.message.chat_id
username = update.message.from_user.username or "Неизвестный пользователь"
if message_text:
message_score = score(message_text)
logger.info("score: %s. message_text: %s", message_score, message_text)
if message_score >= THRESHOLD:
logger.info("-" * 50)
logger.info(f"Обнаружено сообщение с ID {update.message.message_id}, похожее на спам'")
if str(user_id) in whitelist:
logger.info(f"Пользователь {username} в вайтлисте")
else:
# 1. Удаляем сообщение
await update.message.delete()
logger.info(f"Удалено сообщение с ID {update.message.message_id}")
logger.info(update.message.to_json())
# 2. Баним пользователя
try:
await context.bot.ban_chat_member(chat_id=chat_id, user_id=user_id)
logger.info(f"Пользователь @{username} (ID: {user_id}) заблокирован.")
except Exception as e:
logger.error(f"Не удалось заблокировать пользователя @{username} (ID: {user_id}): {e}")
def main():
application = ApplicationBuilder().token(SETTINGS.BOT_TOKEN).build()
application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handle_message))
logger.info("Бот запущен и мониторит чат...")
application.run_polling()
logger.info("Закончили работу...")
if __name__ == "__main__":
main()
import os
from itertools import batched
import joblib
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from transformers import AutoTokenizer, AutoModel
from app.tools import MinioInit
from app.utils import prepare_data, embed
tokenizer = AutoTokenizer.from_pretrained("cointegrated/rubert-tiny2")
model = AutoModel.from_pretrained("cointegrated/rubert-tiny2")
def main():
spam_path = "./data/spam.json"
ham_path = "./data/ham.json"
whitelist_path = "./model/whitelist.json"
assert os.path.exists(whitelist_path)
assert os.path.exists(ham_path)
assert os.path.exists(spam_path)
spam = prepare_data(spam_path)
ham = prepare_data(ham_path)
print(spam.shape, ham.shape)
preds = []
n = spam.shape[0] * 11
train_y = np.hstack([np.zeros(n), np.ones(spam.shape[0])])
for batch in batched(ham, n):
if len(batch) == n:
train_X = np.vstack([np.vstack(batch), spam])
# clf = RandomForestClassifier(random_state=20250409, n_estimators=100)
clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(train_X, train_y)
preds.append(clf.predict(ham))
score_ham = np.mean(preds, axis=0)
print(len(preds))
print((score_ham > 0).sum(), n)
train_X = np.vstack([ham[np.argsort(score_ham)[-n:]], spam])
clf.fit(train_X, train_y)
joblib.dump(clf, './model/spam_ham_n.pkl')
mi = MinioInit()
print("save_file")
with open('./model/spam_ham_n.pkl', 'rb') as f:
mi.save_file(f, 'model/spam_ham_n.pkl')
if os.path.exists('./model/spam_ham_n.pkl'):
os.remove('./model/spam_ham_n.pkl')
print("Загружаем модель")
clf2 = mi.load_data('model/spam_ham_n.pkl')
test = clf2.predict_proba(embed("""От 300 $ в день""").reshape(1, -1))[0, 1]
print('test ', test)
if __name__ == "__main__":
main()
Forwarded from Artem
Forwarded from vitgum
vitgum https://disk.yandex.ru/d/diX5IOiJEFLXmg
Forwarded from Евгений Диденко
Конспект LLM by Data Secrets.pdf29.5 MB
