<?php
/**
* Handler for searching with MySQL's full text search.
*
* @package XenForo_Search
*/
class XenForo_Search_SourceHandler_MySqlFt extends XenForo_Search_SourceHandler_Abstract
{
/**
* Minimum character length of searchable words. Used for error/warning detection.
*
* @var integer
*/
protected $_minWordLength = 4;
protected $_bulkInserts = array();
protected $_bulkInsertLength = 0;
const SPLIT_CHAR_RANGES = '\x00-\x20\x21\x28\x29\x2C-\x2F\x3A-\x40\x5B-\x5E\x60\x7B\x7D-\x7F';
public function __construct()
{
$this->_minWordLength = XenForo_Application::get('options')->searchMinWordLength;
}
/**
* Determines if this source supports relevance sorting.
*
* @return boolean;
*/
public function supportsRelevance()
{
return false;
}
/**
* Inserts or replaces into the index.
*
* @see XenForo_Search_SourceHandler_Abstract::insertIntoIndex()
*/
public function insertIntoIndex($contentType, $contentId, $title, $message, $itemDate, $userId, $discussionId = 0, array $metadata = array())
{
if (utf8_strlen($title) > 250)
{
$originalTitle = $title;
$offset = 250;
$title = utf8_substr($title, 0, $offset);
if ($pos = utf8_strpos($title, ' ', $offset - 15))
{
$title = utf8_substr($title, 0, $pos);
$offset = $pos;
}
$message .= ' ' . utf8_substr($originalTitle, $offset);
$title = trim($title);
$message = trim($message);
}
$metadataPieces = array(
$this->getMetadataKey('user', $userId),
$this->getMetadataKey('content', $contentType)
);
foreach ($metadata AS $metadataKey => $value)
{
$piece = $this->getMetadataKey($metadataKey, $value);
if (is_array($piece))
{
$metadataPieces = array_merge($metadataPieces, $piece);
} else
{
$metadataPieces[] = $piece;
}
}
$db = $this->_getDb();
$row = '(' . $db->quote($contentType) . ', ' . $db->quote(intval($contentId))
. ', ' . $db->quote($title) . ', ' . $db->quote($message)
. ', ' . $db->quote(implode(' ', $metadataPieces))
. ', ' . $db->quote(intval($itemDate)) . ', ' . $db->quote(intval($userId))
. ', ' . $db->quote(intval($discussionId)) . ')';
if ($this->_isRebuild)
{
$this->_bulkInserts[] = $row;
$this->_bulkInsertLength += strlen($row);
if ($this->_bulkInsertLength > 500000)
{
$this->_pushToIndex($this->_bulkInserts);
$this->_bulkInserts = array();
$this->_bulkInsertLength = 0;
}
}
else
{
$this->_pushToIndex($row);
}
}
/**
* Runs the actual query to replace/update the index.
*
* @param string|array $record A record (SQL) or array of SQL
*/
protected function _pushToIndex($record)
{
if (is_array($record))
{
$record = implode(',', $record);
}
if (!$record)
{
return;
}
$this->_getDb()->query('
REPLACE ' . ($this->_isRebuild || !XenForo_Application::get('options')->enableInsertDelayed ? '' : 'DELAYED') . ' INTO xf_search_index
(content_type, content_id,
title, message, metadata,
item_date, user_id, discussion_id)
VALUES
' . $record
);
}
/**
* When rebuilding, it might be advantageous to bulk update records. This function
* must be called to ensure that all records are updated together.
*/
public function finalizeRebuildSet()
{
$this->_pushToIndex($this->_bulkInserts);
}
/**
* Updates a record in the index.
*
* @see XenForo_Search_SourceHandler_Abstract::updateIndex()
*/
public function updateIndex($contentType, $contentId, array $fieldUpdates)
{
$db = $this->_getDb();
$db->update('xf_search_index',
$fieldUpdates,
'content_type = ' . $db->quote($contentType) . ' AND content_id = ' . $db->quote($contentId)
);
}
/**
* Deletes one or more record from the index.
*
* @see XenForo_Search_SourceHandler_Abstract::deleteFromIndex()
*/
public function deleteFromIndex($contentType, array $contentIds)
{
if (!$contentIds)
{
return;
}
$db = $this->_getDb();
$db->delete('xf_search_index',
'content_type = ' . $db->quote($contentType) . ' AND content_id IN (' . $db->quote($contentIds) . ')'
);
}
/**
* Deletes the entire search index or a particular part of it.
*
* @param string|null $contentType If specified, only deletes the index for this type
*/
public function deleteIndex($contentType = null)
{
if (!$contentType)
{
XenForo_Application::getDb()->query('TRUNCATE TABLE xf_search_index');
}
}
/**
* Executes a search against the full text index.
*
* @param string $searchQuery Text to search for
* @param boolean $titleOnly If true, only searches text in titles
* @param array $processedConstraints Structured constraints
* @param array $orderParts Structured ordered by parts
* @param string $groupByDiscussionType If grouping, content type of grouped results
* @param integer $maxResults
* @param XenForo_Search_DataHandler_Abstract $typeHandler Type-specific handler, for joins
*
* @return array Search results ([] => array(content type, id))
*/
public function executeSearch($searchQuery, $titleOnly, array $processedConstraints, array $orderParts,
$groupByDiscussionType, $maxResults, XenForo_Search_DataHandler_Abstract $typeHandler = null
)
{
$db = $this->_getDb();
$queryParts = $this->tokenizeQuery($searchQuery);
$searchQuery = $this->parseTokenizedQuery($queryParts, $processedConstraints);
if ($titleOnly)
{
$matchFields = 'search_index.title, search_index.metadata';
}
else
{
$matchFields = 'search_index.title, search_index.message, search_index.metadata';
}
$tables = array();
$whereClauses = array();
foreach ($processedConstraints AS $constraint)
{
if (isset($constraint['query']) && !isset($constraint['metadata']))
{
// pull queries without metadata alternatives
list($queryTable, $queryField, $queryOperator, $queryValues) = $constraint['query'];
if (is_array($queryValues) && count($queryValues) == 0)
{
continue;
}
if ($queryOperator == '=' && is_array($queryValues))
{
$whereClauses[] = "$queryTable.$queryField IN (" . $db->quote($queryValues) . ")";
}
else
{
if (!is_scalar($queryValues))
{
$queryValues = strval($queryValues);
}
$whereClauses[] = "$queryTable.$queryField $queryOperator " . $db->quote($queryValues);
}
$tables[] = $queryTable;
}
}
$orderFields = array();
foreach ($orderParts AS $order)
{
list($orderTable, $orderField, $orderDirection) = $order;
$orderFields[] = "$orderTable.$orderField $orderDirection";
$tables[] = $orderTable;
}
$orderClause = ($orderFields ? 'ORDER BY ' . implode(', ', $orderFields) : 'ORDER BY NULL');
$tables = array_flip($tables);
unset($tables['search_index']);
if ($typeHandler)
{
$joinStructures = $typeHandler->getJoinStructures($tables);
$joins = array();
foreach ($joinStructures AS $tableAlias => $joinStructure)
{
list($relationshipTable, $relationshipField) = $joinStructure['relationship'];
$joins[] = "INNER JOIN $joinStructure[table] AS $tableAlias ON
($tableAlias.$joinStructure[key] = $relationshipTable.$relationshipField)";
}
}
else
{
$joins = array();
}
$extraWhere = ($whereClauses ? 'AND (' . implode(') AND (', $whereClauses) . ')' : '');
if ($groupByDiscussionType)
{
$selectFields = $db->quote($groupByDiscussionType) . ' AS content_type, search_index.discussion_id AS content_id';
$groupByClause = 'GROUP BY search_index.discussion_id';
}
else
{
$selectFields = 'search_index.content_type, search_index.content_id';
$groupByClause = '';
}
if ($maxResults < 1)
{
$maxResults = 100;
}
$maxResults = intval($maxResults);
if ($this->_searcher && $this->_searcher->hasErrors())
{
return array();
}
return $db->fetchAll("
SELECT $selectFields
FROM xf_search_index AS search_index
" . implode("\n", $joins) . "
WHERE MATCH($matchFields) AGAINST (? IN BOOLEAN MODE)
$extraWhere
$groupByClause
$orderClause
LIMIT $maxResults
", $searchQuery, Zend_Db::FETCH_NUM);
}
/**
* Searches for content by the specified user ID.
*
* @see XenForo_Search_SourceHandler_Abstract::executeSearchByUserId()
*/
public function executeSearchByUserId($userId, $maxDate, $maxResults)
{
if ($maxResults < 1)
{
$maxResults = 100;
}
$maxResults = intval($maxResults);
$db = $this->_getDb();
if ($maxDate)
{
$dateCutOff = 'AND search_index.item_date < ' . $db->quote($maxDate);
}
else
{
$dateCutOff = '';
}
return $db->fetchAll($db->limit(
'
SELECT search_index.content_type, search_index.content_id
FROM xf_search_index AS search_index
WHERE search_index.user_id = ?
' . $dateCutOff . '
ORDER BY search_index.item_date DESC
', $maxResults
), $userId, Zend_Db::FETCH_NUM);
}
/**
* Gets the string form of a piece of metadata.
*
* @param string $keyName Type of metadata
* @param string|array $value Metadata value; if an array, gets metadata for each value
*
* @return string|array String if $value was a string, array if $value was an array
*/
public function getMetadataKey($keyName, $value)
{
if (is_array($value))
{
$output = array();
foreach ($value AS $childValue)
{
$output[] = '_md_' . $keyName . '_' . preg_replace('/[^a-z0-9_]/i', '', strval($childValue));
}
return $output;
}
else
{
return '_md_' . $keyName . '_' . preg_replace('/[^a-z0-9_]/i', '', $value);
}
}
/**
* Tokenizes a search query into parts.
*
* @param string $query
*
* @return array Tokenized query, [] => array(0 => modifier (empty, +, -, |), 1 => term). Term will include ".." if given.
*/
public function tokenizeQuery($query)
{
$query = str_replace(array('(', ')'), '', trim($query)); // don't support grouping yet
preg_match_all('/
(?<=[' . self::SPLIT_CHAR_RANGES .'\-\+\|]|^)
(?P<modifier>\-(?!\s)|\+|\||)
[' . self::SPLIT_CHAR_RANGES .']*
(?P<term>"(?P<quoteTerm>[^"]+)"|[^' . self::SPLIT_CHAR_RANGES .'\-\+\|]+)
/ix', $query, $matches, PREG_SET_ORDER);
$output = array();
$i = 0;
$haveWords = false;
$invalidWords = array();
foreach ($matches AS $match)
{
$iStart = $i;
if ($match['modifier'] == '|' && $i > 0 && $output[$i - 1][0] == '')
{
$output[$i - 1][0] = '|';
}
else if ($match['modifier'] == '|' && $i == 0)
{
$match['modifier'] = '';
}
$output[$i] = array($match['modifier'], $match['term']);
if (!empty($match['quoteTerm']))
{
$words = $this->splitWords($match['quoteTerm']);
}
else
{
$words = $this->splitWords($match['term']);
}
foreach ($words AS $word)
{
if ($word === '')
{
continue;
}
if (utf8_strlen($word) < $this->_minWordLength)
{
$invalidWords[] = $word;
}
else if (in_array($word, self::$stopWords))
{
$invalidWords[] = $word;
}
else
{
$haveWords = true;
}
}
$i++;
}
if (!$haveWords)
{
if ($invalidWords)
{
$this->error(new XenForo_Phrase('search_could_not_be_completed_because_search_keywords_were_too'), 'keywords');
}
else
{
//$this->error(new XenForo_Phrase('please_enter_valid_search_query'), 'keywords');
}
}
else if ($invalidWords)
{
$this->warning(
new XenForo_Phrase(
'following_words_were_not_included_in_your_search_x',
array('words' => implode(', ', $invalidWords))
), 'keywords'
);
}
return $output;
}
/**
* Split words by the delimiters used by MySQL.
*
* @param string $words
*
* @return array
*/
public function splitWords($words)
{
// delimiters: 0 - 38, 40, 41, 43 - 47, 58 - 64, 91 - 94, 96, 123 - 127
return preg_split('/[' . self::SPLIT_CHAR_RANGES . ']/', $words, -1, PREG_SPLIT_NO_EMPTY);
}
/**
* Parses a tokenized query into the final query to give to MySQL. Metadata
* will also be searched in the query.
*
* @param array $query Tokenized query
* @param array $processedConstraints Constraints, in source-agnostic format
*
* @return string Query ready for MySQL
*/
public function parseTokenizedQuery(array $query, array $processedConstraints)
{
$output = '';
foreach ($query AS $part)
{
if ($part[0] == '')
{
$part[0] = '+';
}
else if ($part[0] == '|')
{
$part[0] = ''; // default in mysql
}
$output .= ' ' . $part[0] . $part[1];
}
foreach ($processedConstraints AS $constraint)
{
if (!isset($constraint['metadata']))
{
continue;
}
list($metadataField, $metadataValues) = $constraint['metadata'];
$metadata = $this->getMetadataKey($metadataField, $metadataValues);
if (is_array($metadata))
{
$output .= ' +(' . implode(' ', $metadata) . ')';
}
else
{
$output .= ' +' . $metadata;
}
}
return trim($output);
}
public static $stopWords = array(
'a\'s', 'able', 'about', 'above', 'according', 'accordingly', 'across', 'actually',
'after', 'afterwards', 'again', 'against', 'ain\'t', 'all', 'allow', 'allows',
'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'am',
'among', 'amongst', 'an', 'and', 'another', 'any', 'anybody', 'anyhow',
'anyone', 'anything', 'anyway', 'anyways', 'anywhere', 'apart', 'appear', 'appreciate',
'appropriate', 'are', 'aren\'t', 'around', 'as', 'aside', 'ask', 'asking',
'associated', 'at', 'available', 'away', 'awfully', 'be', 'became', 'because',
'become', 'becomes', 'becoming', 'been', 'before', 'beforehand', 'behind', 'being',
'believe', 'below', 'beside', 'besides', 'best', 'better', 'between', 'beyond',
'both', 'brief', 'but', 'by', 'c\'mon', 'c\'s', 'came', 'can',
'can\'t', 'cannot', 'cant', 'cause', 'causes', 'certain', 'certainly', 'changes',
'clearly', 'co', 'com', 'come', 'comes', 'concerning', 'consequently', 'consider',
'considering', 'contain', 'containing', 'contains', 'corresponding', 'could', 'couldn\'t', 'course',
'currently', 'definitely', 'described', 'despite', 'did', 'didn\'t', 'different', 'do',
'does', 'doesn\'t', 'doing', 'don\'t', 'done', 'down', 'downwards', 'during',
'each', 'edu', 'eg', 'eight', 'either', 'else', 'elsewhere', 'enough',
'entirely', 'especially', 'et', 'etc', 'even', 'ever', 'every', 'everybody',
'everyone', 'everything', 'everywhere', 'ex', 'exactly', 'example', 'except', 'far',
'few', 'fifth', 'first', 'five', 'followed', 'following', 'follows', 'for',
'former', 'formerly', 'forth', 'four', 'from', 'further', 'furthermore', 'get',
'gets', 'getting', 'given', 'gives', 'go', 'goes', 'going', 'gone',
'got', 'gotten', 'greetings', 'had', 'hadn\'t', 'happens', 'hardly', 'has',
'hasn\'t', 'have', 'haven\'t', 'having', 'he', 'he\'s', 'hello', 'help',
'hence', 'her', 'here', 'here\'s', 'hereafter', 'hereby', 'herein', 'hereupon',
'hers', 'herself', 'hi', 'him', 'himself', 'his', 'hither', 'hopefully',
'how', 'howbeit', 'however', 'i\'d', 'i\'ll', 'i\'m', 'i\'ve', 'ie',
'if', 'ignored', 'immediate', 'in', 'inasmuch', 'inc', 'indeed', 'indicate',
'indicated', 'indicates', 'inner', 'insofar', 'instead', 'into', 'inward', 'is',
'isn\'t', 'it', 'it\'d', 'it\'ll', 'it\'s', 'its', 'itself', 'just',
'keep', 'keeps', 'kept', 'know', 'known', 'knows', 'last', 'lately',
'later', 'latter', 'latterly', 'least', 'less', 'lest', 'let', 'let\'s',
'like', 'liked', 'likely', 'little', 'look', 'looking', 'looks', 'ltd',
'mainly', 'many', 'may', 'maybe', 'me', 'mean', 'meanwhile', 'merely',
'might', 'more', 'moreover', 'most', 'mostly', 'much', 'must', 'my',
'myself', 'name', 'namely', 'nd', 'near', 'nearly', 'necessary', 'need',
'needs', 'neither', 'never', 'nevertheless', 'new', 'next', 'nine', 'no',
'nobody', 'non', 'none', 'noone', 'nor', 'normally', 'not', 'nothing',
'novel', 'now', 'nowhere', 'obviously', 'of', 'off', 'often', 'oh',
'ok', 'okay', 'old', 'on', 'once', 'one', 'ones', 'only',
'onto', 'or', 'other', 'others', 'otherwise', 'ought', 'our', 'ours',
'ourselves', 'out', 'outside', 'over', 'overall', 'own', 'particular', 'particularly',
'per', 'perhaps', 'placed', 'please', 'plus', 'possible', 'presumably', 'probably',
'provides', 'que', 'quite', 'qv', 'rather', 'rd', 're', 'really',
'reasonably', 'regarding', 'regardless', 'regards', 'relatively', 'respectively', 'right', 'said',
'same', 'saw', 'say', 'saying', 'says', 'second', 'secondly', 'see',
'seeing', 'seem', 'seemed', 'seeming', 'seems', 'seen', 'self', 'selves',
'sensible', 'sent', 'serious', 'seriously', 'seven', 'several', 'shall', 'she',
'should', 'shouldn\'t', 'since', 'six', 'so', 'some', 'somebody', 'somehow',
'someone', 'something', 'sometime', 'sometimes', 'somewhat', 'somewhere', 'soon', 'sorry',
'specified', 'specify', 'specifying', 'still', 'sub', 'such', 'sup', 'sure',
't\'s', 'take', 'taken', 'tell', 'tends', 'th', 'than', 'thank',
'thanks', 'thanx', 'that', 'that\'s', 'thats', 'the', 'their', 'theirs',
'them', 'themselves', 'then', 'thence', 'there', 'there\'s', 'thereafter', 'thereby',
'therefore', 'therein', 'theres', 'thereupon', 'these', 'they', 'they\'d', 'they\'ll',
'they\'re', 'they\'ve', 'think', 'third', 'this', 'thorough', 'thoroughly', 'those',
'though', 'three', 'through', 'throughout', 'thru', 'thus', 'to', 'together',
'too', 'took', 'toward', 'towards', 'tried', 'tries', 'truly', 'try',
'trying', 'twice', 'two', 'un', 'under', 'unfortunately', 'unless', 'unlikely',
'until', 'unto', 'up', 'upon', 'us', 'use', 'used', 'useful',
'uses', 'using', 'usually', 'value', 'various', 'very', 'via', 'viz',
'vs', 'want', 'wants', 'was', 'wasn\'t', 'way', 'we', 'we\'d',
'we\'ll', 'we\'re', 'we\'ve', 'welcome', 'well', 'went', 'were', 'weren\'t',
'what', 'what\'s', 'whatever', 'when', 'whence', 'whenever', 'where', 'where\'s',
'whereafter', 'whereas', 'whereby', 'wherein', 'whereupon', 'wherever', 'whether', 'which',
'while', 'whither', 'who', 'who\'s', 'whoever', 'whole', 'whom', 'whose',
'why', 'will', 'willing', 'wish', 'with', 'within', 'without', 'won\'t',
'wonder', 'would', 'wouldn\'t', 'yes', 'yet', 'you', 'you\'d', 'you\'ll',
'you\'re', 'you\'ve', 'your', 'yours', 'yourself', 'yourselves', 'zero'
);
}