jenkins-bot has submitted this change and it was merged.
Change subject: Add list of highlighted words and total term frequency to API
output
......................................................................
Add list of highlighted words and total term frequency to API output
Bug: T104443
Change-Id: I56593181890c79afbdb69f85e83c9a3a6c5c9a14
---
M includes/Api/ApiFlowSearch.php
M includes/Search/SearchEngine.php
M includes/Search/Searcher.php
3 files changed, 163 insertions(+), 16 deletions(-)
Approvals:
Matthias Mullie: Looks good to me, approved
jenkins-bot: Verified
diff --git a/includes/Api/ApiFlowSearch.php b/includes/Api/ApiFlowSearch.php
index bdb2002..f3896b6 100644
--- a/includes/Api/ApiFlowSearch.php
+++ b/includes/Api/ApiFlowSearch.php
@@ -10,6 +10,7 @@
use Flow\Model\UUID;
use Flow\Search\Connection;
use Flow\Search\SearchEngine;
+use Flow\Search\Searcher;
use Flow\TalkpageManager;
use MWNamespace;
use Status;
@@ -28,7 +29,10 @@
public function execute() {
$params = $this->extractRequestParams();
- $this->searchEngine->setType( $params['type'] );
+ if ( $params['type'] ) {
+ $this->searchEngine->setType( $params['type'] );
+ }
+
$this->searchEngine->setLimitOffset( $params['limit'],
$params['offset'] );
$this->searchEngine->setSort( $params['sort'] );
@@ -55,10 +59,31 @@
throw new InvalidDataException( $status->getMessage(),
'fail-search' );
}
- /** @var \Elastica\ResultSet|null $result */
- $result = $status->getValue();
- // result can be null, if nothing was found
- $results = $result === null ? array() : $result->getResults();
+ /** @var \Elastica\ResultSet|null $resultSet */
+ $resultSet = $status->getValue();
+ // $resultSet can be null, if nothing was found
+ $results = $resultSet === null ? array() :
$resultSet->getResults();
+
+ // list of highlighted words
+ $highlights = array();
+ /** @var \Elastica\Result $result */
+ foreach ( $results as $result ) {
+ // there'll always be exactly 1 excerpt
+ // see Searcher.php, ...->setHighlight() config
+ $excerpt = $result->getHighlights();
+ $excerpt = $excerpt[Searcher::HIGHLIGHT_FIELD][0];
+
+ $pre = preg_quote( Searcher::HIGHLIGHT_PRE, '/' );
+ $post = preg_quote( Searcher::HIGHLIGHT_POST, '/' );
+ if ( preg_match_all( '/' . $pre . '(.*?)' . $post .
'/', $excerpt, $matches ) ) {
+ $highlights += array_flip( $matches[1] );
+ }
+ }
+ $highlights = array_keys( $highlights );
+
+ // total term frequency
+ $ttf = $resultSet->getAggregation( 'ttf' );
+ $ttf = $ttf['value'];
$topicIds = array();
foreach ( $results as $topic ) {
@@ -68,7 +93,9 @@
// output similar to view-topiclist
$results = $this->formatApi( $topicIds );
// search-specific output
- $results['total'] = $result->getTotalHits();
+ $results['total'] = $resultSet->getTotalHits();
+ $results['highlights'] = $highlights;
+ $results['ttf'] = $ttf;
$this->getResult()->addValue( null, $this->getModuleName(),
$results );
}
diff --git a/includes/Search/SearchEngine.php b/includes/Search/SearchEngine.php
index cdc3953..f238bf9 100644
--- a/includes/Search/SearchEngine.php
+++ b/includes/Search/SearchEngine.php
@@ -42,7 +42,7 @@
/**
* @param string $term text to search
- * @return Status
+ * @return \Status
*/
public function searchText( $term ) {
$term = trim( $term );
diff --git a/includes/Search/Searcher.php b/includes/Search/Searcher.php
index cf1c249..2930bde 100644
--- a/includes/Search/Searcher.php
+++ b/includes/Search/Searcher.php
@@ -5,10 +5,16 @@
use Elastica\Query;
use Elastica\Query\QueryString;
use Elastica\Exception\ExceptionInterface;
+use Elastica\Request;
+use Elastica\ResultSet;
use PoolCounterWorkViaCallback;
use Status;
class Searcher {
+ const HIGHLIGHT_FIELD = 'revisions.text';
+ const HIGHLIGHT_PRE = '<span class="searchmatch">';
+ const HIGHLIGHT_POST = '</span>';
+
/**
* @var string|false $type
*/
@@ -47,6 +53,26 @@
$queryString->setFields( array( 'revisions.text' ) );
$this->query->setQuery( $queryString );
+ // add aggregation to determine exact amount of matching search
terms
+ $terms = $this->getTerms( $term );
+ $this->query->addAggregation( $this->termsAggregation( $terms )
);
+
+ // @todo: abstract-away this config? (core/cirrus also has this
- share it somehow?)
+ $this->query->setHighlight( array(
+ 'fields' => array(
+ static::HIGHLIGHT_FIELD => array(
+ 'type' => 'plain',
+ 'order' => 'score',
+
+ // we want just 1 excerpt of result
text, which includes all highlights
+ 'number_of_fragments' => 1,
+ 'fragment_size' => 10000, // We want
the whole value but more than this is crazy
+ ),
+ ),
+ 'pre_tags' => array( static::HIGHLIGHT_PRE ),
+ 'post_tags' => array( static::HIGHLIGHT_POST ),
+ ) );
+
// @todo: support insource: queries (and perhaps others)
$searchable = Connection::getFlowIndex( $this->indexBaseName );
@@ -61,22 +87,116 @@
// Perform the search
$work = new PoolCounterWorkViaCallback( 'Flow-Search',
"_elasticsearch", array(
'doWork' => function() use ( $search ) {
- try {
- $result = $search->search();
- return Status::newGood( $result
);
- } catch ( ExceptionInterface $e ) {
- return Status::newFatal(
'flow-error-search' );
+ try {
+ $result = $search->search();
+ return Status::newGood( $result );
+ } catch ( ExceptionInterface $e ) {
+ if ( strpos( $e->getMessage(), 'dynamic
scripting for [groovy] disabled' ) ) {
+ // known issue with default ES
config, let's display a more helpful message
+ return Status::newFatal( new
\RawMessage(
+ "Couldn't complete
search: dynamic scripting needs to be enabled. " .
+ "Please add
'script.disable_dynamic: false' to your elasticsearch.yml"
+ ) );
}
- },
- 'error' => function( $status ) {
- $status = $status->getErrorsArray();
- wfLogWarning( 'Pool error searching
Elasticsearch: ' . $status[ 0 ][ 0 ] );
+
return Status::newFatal(
'flow-error-search' );
}
+ },
+ 'error' => function( Status $status ) {
+ $status = $status->getErrorsArray();
+ wfLogWarning( 'Pool error searching
Elasticsearch: ' . $status[0][0] );
+ return Status::newFatal( 'flow-error-search' );
+ }
) );
$result = $work->execute();
return $result;
}
+
+ /**
+ * We want to retrieve the total amount of search word hits
+ * (static::termsAggregation) but our search terms may not be how
+ * ElasticSearch stores the words in its index.
+ * Elastic will "analyze" text (perform stemming, etc) and store
+ * the terms in a normalized way.
+ * AFAICT, there is not really a way to get to that information
+ * from within a search query.
+ *
+ * Luckily, since 1.0, Elastic supports _termvector, which gives
+ * you statistics about the terms in your document.
+ * Since 1.4, Elastic supports feeding _termvector documents to
+ * analyze.
+ * We're going to (ab)use this by letting it respond with term
+ * information on a bogus document that contains only our current
+ * search terms.
+ * So we'll give it a document with just our keywords for the
+ * column that we're searching in (revisions.text) and Elastic will
+ * use that column's configuration to analyze the text we feed it.
+ * It will then respond with the normalized terms & their stats.
+ *
+ * @param string $terms
+ * @return array
+ */
+ protected function getTerms( $terms ) {
+ $terms = preg_split( '/\s+/', $terms );
+
+ // _termvectors only works on a type, but our types are
+ // configured exactly the same so it doesn't matter which
+ $types = Connection::getAllTypes();
+ $searchable = Connection::getFlowIndex( $this->indexBaseName );
+ $searchable = $searchable->getType( array_pop( $types ) );
+
+ $query = array(
+ // bogus document that contains the current search term
+ 'doc' => array(
+ 'revisions' => array(
+ 'text' => $terms,
+ ),
+ ),
+ "fields" => array( "revisions.text" ),
+ );
+
+ // Elastica has no abstraction over _termvector like it has
+ // for _query, so just do the request ourselves
+ $response = $searchable->request(
+ '_termvector',
+ Request::POST,
+ $query,
+ array()
+ );
+
+ $data = $response->getData();
+ return array_keys(
$data['term_vectors']['revisions.text']['terms'] );
+ }
+
+ /**
+ * We can only do this if dynamic scripting is enabled. In
elasticsearch.yml:
+ * script.disable_dynamic: false
+ * @see vendor/ruffin/elastica/test/bin/run_elasticsearch.sh
+ *
+ * @param array $terms
+ * @return \Elastica\Aggregation\Sum
+ */
+ protected function termsAggregation( array $terms ) {
+ $terms = str_replace( '"', '\\"', $terms );
+
+ $script = '
+keywords = ["' . implode( '","', $terms ) . '"]
+total = 0
+for (term in keywords) {
+ total += _index["revisions.text"][term].tf()
+}
+return total';
+ $script = new \Elastica\Script( $script, null, 'groovy' );
+
+ $aggregation = new \Elastica\Aggregation\Sum( 'ttf' );
+ // $aggregation->setScript() doesn't seem to properly set
'lang': 'groovy'
+ // see https://github.com/ruflin/Elastica/pull/748
+ // $aggregation->setScript( $script );
+ $aggregation->setParams( array( 'lang' => 'groovy' ) );
+ $aggregation->setParam( 'script', $script->getScript() );
+
+ return $aggregation;
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/184404
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I56593181890c79afbdb69f85e83c9a3a6c5c9a14
Gerrit-PatchSet: 14
Gerrit-Project: mediawiki/extensions/Flow
Gerrit-Branch: master
Gerrit-Owner: Matthias Mullie <[email protected]>
Gerrit-Reviewer: EBernhardson <[email protected]>
Gerrit-Reviewer: Etonkovidova <[email protected]>
Gerrit-Reviewer: Legoktm <[email protected]>
Gerrit-Reviewer: Manybubbles <[email protected]>
Gerrit-Reviewer: Mattflaschen <[email protected]>
Gerrit-Reviewer: Matthias Mullie <[email protected]>
Gerrit-Reviewer: SG <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
