from llama_index.embeddings.huggingface import HuggingFaceEmbedding
from llama_index.core import VectorStoreIndex
from llama_index.core.node_parser import SentenceSplitter
from llama_index.core import SimpleDirectoryReader
from llama_index.core import Settings
HF_TOKEN = "Ваш токен на Hugging Face"
embeddings = HuggingFaceEmbedding(
model_name="google/embeddinggemma-300m",
query_instruction="task: search result | query: ",
text_instruction="title: none | text: ",
token=settings.HF_TOKEN,
)
Settings.embed_model = embeddings
documents = SimpleDirectoryReader("./data").load_data()
splitter = SentenceSplitter(chunk_size=512, chunk_overlap=256)
index = VectorStoreIndex.from_documents(
documents, transformations=[splitter], embed_model=embeddings
)
index.storage_context.persist(persist_dir="./vector")
Post #1637
313
2) векторный индекс, максимально тупо
- ❤ 1