Поиск по небольшой базе знаний

Функция обучения

function trainSearchModel(samples) {
  if (!Array.isArray(samples) || samples.length < 3 || samples.length > 12) throw new Error('SEARCH_COUNT');
  const frequency = new Map();
  const documents = samples.map(sample => {
    if (!sample || typeof sample.title !== 'string' || !sample.title.trim() ||
        typeof sample.text !== 'string' || !sample.text.trim()) throw new Error('TRAINING_FORMAT');
    const tokens = tokenizeText(sample.title + ' ' + sample.text);
    if (!tokens.length) throw new Error('TRAINING_WORDS');
    new Set(tokens).forEach(token => frequency.set(token, (frequency.get(token) || 0) + 1));
    return {title: sample.title.trim(), text: sample.text.trim(), tokens};
  });
  const idf = new Map();
  frequency.forEach((count, token) => idf.set(token, Math.log((1 + samples.length) / (1 + count)) + 1));
  for (const document of documents) {
    const vector = new Map();
    document.tokens.forEach(token => vector.set(token, (vector.get(token) || 0) + 1));
    vector.forEach((count, token) => vector.set(token, count * idf.get(token)));
    document.vector = vector;
    document.norm = Math.sqrt(Array.from(vector.values()).reduce((sum, weight) => sum + weight * weight, 0));
    delete document.tokens;
  }
  return {kind: 'search', documents, idf, vocabulary: new Set(idf.keys())};
}

Функция проверки запроса

function searchKnowledge(model, text) {
  if (!model || model.kind !== 'search') throw new Error('NOT_TRAINED');
  if (typeof text !== 'string' || text.length > 500) throw new Error('QUERY_LIMIT');
  const tokens = tokenizeText(text);
  const known = tokens.filter(token => model.vocabulary.has(token));
  const coverage = tokens.length ? known.length / tokens.length : 0;
  if (!known.length) return {review: true, reason: tokens.length ? 'UNKNOWN_WORDS' : 'QUERY_WORDS', document: null, scores: [], known: 0, total: tokens.length, coverage, gap: 0};

  const query = new Map();
  known.forEach(token => query.set(token, (query.get(token) || 0) + 1));
  query.forEach((count, token) => query.set(token, count * model.idf.get(token)));
  const norm = Math.sqrt(Array.from(query.values()).reduce((sum, weight) => sum + weight * weight, 0));
  const scores = model.documents.map((document, index) => {
    let product = 0;
    query.forEach((weight, token) => { product += weight * (document.vector.get(token) || 0); });
    const score = Math.min(1, Math.max(0, product / (norm * document.norm)));
    return {label: document.title, score, index};
  }).sort((left, right) => right.score - left.score);
  const gap = scores[0].score - scores[1].score;
  const reason = coverage < 0.2 ? 'LOW_COVERAGE' : scores[0].score < 0.2 ? 'WEAK_MATCH' : gap < 0.05 ? 'CLOSE_SCORES' : '';
  return {review: !!reason, reason, document: reason ? null : model.documents[scores[0].index], scores, known: known.length, total: tokens.length, coverage, gap};
}

Показаны функции из этой демонстрации. Полная реализация также включает подготовку текста и интерфейс.