MaxSem has submitted this change and it was merged. Change subject: Initial conversion of prop=extracts from MobileFrontend ......................................................................
Initial conversion of prop=extracts from MobileFrontend Change-Id: I8a02f38554f6f06a00e6229889f7545f8ee8976a --- A ApiQueryExtracts.php A ExtractFormatter.php A ExtractFormatterTest.php A TextExtracts.i18n.php A TextExtracts.php 5 files changed, 596 insertions(+), 0 deletions(-) Approvals: MaxSem: Verified Awjrichards: Looks good to me, approved diff --git a/ApiQueryExtracts.php b/ApiQueryExtracts.php new file mode 100644 index 0000000..fa315bb --- /dev/null +++ b/ApiQueryExtracts.php @@ -0,0 +1,406 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + */ + +class ApiQueryExtracts extends ApiQueryBase { + /** + * @var ParserOptions + */ + private $parserOptions; + private $params; + + public function __construct( $query, $moduleName ) { + parent::__construct( $query, $moduleName, 'ex' ); + } + + public function execute() { + wfProfileIn( __METHOD__ ); + $titles = $this->getPageSet()->getGoodTitles(); + if ( count( $titles ) == 0 ) { + wfProfileOut( __METHOD__ ); + return; + } + $isXml = $this->getMain()->isInternalMode() || $this->getMain()->getPrinter()->getFormat() == 'XML'; + $result = $this->getResult(); + $params = $this->params = $this->extractRequestParams(); + $this->requireMaxOneParameter( $params, 'chars', 'sentences' ); + $continue = 0; + $limit = intval( $params['limit'] ); + if ( $limit > 1 && !$params['intro'] ) { + $limit = 1; + $result->setWarning( "exlimit was too large for a whole article extracts request, lowered to $limit" ); + } + if ( isset( $params['continue'] ) ) { + $continue = intval( $params['continue'] ); + if ( $continue < 0 || $continue > count( $titles ) ) { + $this->dieUsageMsg( '_badcontinue' ); + } + $titles = array_slice( $titles, $continue, null, true ); + } + $count = 0; + /** @var Title $t */ + foreach ( $titles as $id => $t ) { + if ( ++$count > $limit ) { + $this->setContinueEnumParameter( 'continue', $continue + $count - 1 ); + break; + } + $text = $this->getExtract( $t ); + $pageName = $t->getText(); + $text = $this->truncate( $text, $pageName ); + if ( $this->params['plaintext'] ) { + $text = $this->doSections( $text ); + } + + if ( $isXml ) { + $fit = $result->addValue( array( 'query', 'pages', $id ), 'extract', array( '*' => $text ) ); + } else { + $fit = $result->addValue( array( 'query', 'pages', $id ), 'extract', $text ); + } + if ( !$fit ) { + $this->setContinueEnumParameter( 'continue', $continue + $count - 1 ); + break; + } + } + wfProfileOut( __METHOD__ ); + } + + /** + * OpenSearchXml hook handler + * @param array $results + * @return bool + */ + public static function onOpenSearchXml( &$results ) { + global $wgExtractsExtendOpenSearchXml; + if ( !$wgExtractsExtendOpenSearchXml || !count( $results ) ) { + return true; + } + $pageIds = array_keys( $results ); + $api = new ApiMain( new FauxRequest( + array( + 'action' => 'query', + 'prop' => 'extracts', + 'explaintext' => true, + 'exintro' => true, + 'exlimit' => count( $results ), + 'pageids' => implode( '|', $pageIds ), + ) ) + ); + $api->execute(); + $data = $api->getResultData(); + foreach ( $pageIds as $id ) { + if ( isset( $data['query']['pages'][$id]['extract']['*'] ) ) { + $results[$id]['extract'] = $data['query']['pages'][$id]['extract']['*']; + $results[$id]['extract trimmed'] = false; + } + } + return true; + } + + /** + * Returns a processed, but not trimmed extract + * @param Title $title + * @return string + */ + private function getExtract( Title $title ) { + wfProfileIn( __METHOD__ ); + $page = WikiPage::factory( $title ); + + $introOnly = $this->params['intro']; + $text = $this->getFromCache( $page, $introOnly ); + // if we need just first section, try retrieving full page and getting first section out of it + if ( $text === false && $introOnly ) { + $text = $this->getFromCache( $page, false ); + if ( $text !== false ) { + $text = $this->getFirstSection( $text, $this->params['plaintext'] ); + } + } + if ( $text === false ) { + $text = $this->parse( $page ); + $text = $this->convertText( $text, $title, $this->params['plaintext'] ); + $this->setCache( $page, $text ); + } + wfProfileOut( __METHOD__ ); + return $text; + } + + private function cacheKey( WikiPage $page, $introOnly ) { + return wfMemcKey( 'mf', 'extract', $page->getLatest(), $this->params['plaintext'], $introOnly ); + } + + private function getFromCache( WikiPage $page, $introOnly ) { + global $wgMemc; + + $key = $this->cacheKey( $page, $introOnly ); + return $wgMemc->get( $key ); + } + + private function setCache( WikiPage $page, $text ) { + global $wgMemc; + + $key = $this->cacheKey( $page, $this->params['intro'] ); + $wgMemc->set( $key, $text ); + } + + private function getFirstSection( $text, $plainText ) { + if ( $plainText ) { + $regexp = '/^(.*?)(?=' . ExtractFormatter::SECTION_MARKER_START . ')/s'; + } else { + $regexp = '/^(.*?)(?=<h[1-6]\b)/s'; + } + if ( preg_match( $regexp, $text, $matches ) ) { + $text = $matches[0]; + } + return $text; + } + + /** + * Returns page HTML + * @param WikiPage $page + * @return string + */ + private function parse( WikiPage $page ) { + wfProfileIn( __METHOD__ ); + if ( !$this->parserOptions ) { + $this->parserOptions = new ParserOptions( new User( '127.0.0.1' ) ); + } + // first try finding full page in parser cache + if ( $page->isParserCacheUsed( $this->parserOptions, 0 ) ) { + $pout = ParserCache::singleton()->get( $page, $this->parserOptions ); + if ( $pout ) { + $text = $pout->getText(); + if ( $this->params['intro'] ) { + $text = $this->getFirstSection( $text, false ); + } + wfProfileOut( __METHOD__ ); + return $text; + } + } + $request = array( + 'action' => 'parse', + 'page' => $page->getTitle()->getPrefixedText(), + 'prop' => 'text' + ); + if ( $this->params['intro'] ) { + $request['section'] = 0; + } + // in case of cache miss, render just the needed section + $api = new ApiMain( new FauxRequest( $request ) ); + $api->execute(); + $data = $api->getResultData(); + wfProfileOut( __METHOD__ ); + return $data['parse']['text']['*']; + } + + /** + * Converts page HTML into an extract + * @param string $text + * @return string + */ + private function convertText( $text ) { + wfProfileIn( __METHOD__ ); + $fmt = new ExtractFormatter( $text, $this->params['plaintext'], $this->params['sectionformat'] ); + $text = $fmt->getText(); + + wfProfileOut( __METHOD__ ); + return trim( $text ); + } + + /** + * Truncate the given text to a certain number of characters or sentences + * @param string $text The text to truncate + * @param string $pageName Title of the page (for debugging) + * @return string + */ + private function truncate( $text, $pageName ) { + if ( $this->params['chars'] ) { + return $this->getFirstChars( $text, $this->params['chars'] ); + } elseif ( $this->params['sentences'] ) { + return $this->getFirstSentences( $text, $this->params['sentences'], $pageName ); + } + return $text; + } + + /** + * Returns no more than a requested number of characters + * @param string $text + * @param int $requestedLength + * @return string + */ + private function getFirstChars( $text, $requestedLength ) { + wfProfileIn( __METHOD__ ); + $length = mb_strlen( $text ); + if ( $length <= $requestedLength ) { + wfProfileOut( __METHOD__ ); + return $text; + } + $pattern = "#^.{{$requestedLength}}[\\w/]*>?#su"; + preg_match( $pattern, $text, $m ); + $text = $m[0]; + // Fix possibly unclosed tags + $text = $this->tidy( $text ); + $text .= wfMessage( 'ellipsis' )->inContentLanguage()->text(); + wfProfileOut( __METHOD__ ); + return $text; + } + + /** + * @param string $text + * @param int $requestedSentenceCount + * @param string $pageName Title of the page (for debugging) + * @return string + */ + private function getFirstSentences( $text, $requestedSentenceCount, $pageName ) { + wfProfileIn( __METHOD__ ); + // Based on code from OpenSearchXml by Brion Vibber + $endchars = array( + '([^\d])\.\s', '\!\s', '\?\s', // regular ASCII + '。', // full-width ideographic full-stop + '．', '！', '？', // double-width roman forms + '｡', // half-width ideographic full stop + ); + + $endgroup = implode( '|', $endchars ); + $end = "(?:$endgroup)"; + $sentence = ".+?$end+"; + $regexp = "/^($sentence){1,{$requestedSentenceCount}}/u"; + $matches = array(); + $res = preg_match( $regexp, $text, $matches ); + if( $res ) { + $text = $matches[0]; + } else { + if ( $res === false ) { + wfLogWarning( "Regular expresssion compilation failure. Page: $pageName; RegEx: $regexp" ); + } + // Just return the first line + $lines = explode( "\n", $text ); + $text = trim( $lines[0] ); + } + $text = $this->tidy( $text ); + wfProfileOut( __METHOD__ ); + return $text; + } + + /** + * A simple wrapper around tidy + * @param string $text + * @return string + */ + private function tidy( $text ) { + global $wgUseTidy; + + wfProfileIn( __METHOD__ ); + if ( $wgUseTidy && !$this->params['plaintext'] ) { + $text = trim ( MWTidy::tidy( $text ) ); + } + wfProfileOut( __METHOD__ ); + return $text; + } + + private function doSections( $text ) { + $text = preg_replace_callback( + "/" . ExtractFormatter::SECTION_MARKER_START . '(\d)'. ExtractFormatter::SECTION_MARKER_END . "(.*?)$/m", + array( $this, 'sectionCallback' ), + $text + ); + return $text; + } + + private function sectionCallback( $matches ) { + if ( $this->params['sectionformat'] == 'raw' ) { + return $matches[0]; + } + $func = __CLASS__ . "::doSection_{$this->params['sectionformat']}"; + return call_user_func( $func, $matches[1], trim( $matches[2] ) ); + } + + private static function doSection_wiki( $level, $text ) { + $bars = str_repeat( '=', $level ); + return "\n$bars $text $bars"; + } + + private static function doSection_plain( $level, $text ) { + return "\n$text"; + } + + public function getAllowedParams() { + return array( + 'chars' => array( + ApiBase::PARAM_TYPE => 'integer', + ApiBase::PARAM_MIN => 1, + ), + 'sentences' => array( + ApiBase::PARAM_TYPE => 'integer', + ApiBase::PARAM_MIN => 1, + ApiBase::PARAM_MAX => 10, + ), + 'limit' => array( + ApiBase::PARAM_DFLT => 1, + ApiBase::PARAM_TYPE => 'limit', + ApiBase::PARAM_MIN => 1, + ApiBase::PARAM_MAX => 20, + ApiBase::PARAM_MAX2 => 20, + ), + 'intro' => false, + 'plaintext' => false, + 'sectionformat' => array( + ApiBase::PARAM_TYPE => array( 'plain', 'wiki', 'raw' ), + ApiBase::PARAM_DFLT => 'wiki', + ), + 'continue' => array( + ApiBase::PARAM_TYPE => 'integer', + ), + ); + } + + public function getParamDescription() { + return array( + 'chars' => 'How many characters to return, actual text returned might be slightly longer.', + 'sentences' => 'How many sentences to return', + 'limit' => 'How many extracts to return', + 'intro' => 'Return only content before the first section', + 'plaintext' => 'Return extracts as plaintext instead of limited HTML', + 'sectionformat' => array( + 'How to format sections in plaintext mode:', + ' plain - No formatting', + ' wiki - Wikitext-style formatting == like this ==', + " raw - This module's internal representation (secton titles prefixed with <ASCII 1><ASCII 2><section level><ASCII 2><ASCII 1>", + ), + 'continue' => 'When more results are available, use this to continue', + ); + } + + public function getDescription() { + return 'Returns plain-text or limited HTML extracts of the given page(s)'; + } + + public function getPossibleErrors() { + return array_merge( parent::getPossibleErrors(), array( + array( 'code' => '_badcontinue', 'info' => 'Invalid continue param. You should pass the original value returned by the previous query' ), + ) ); + } + + public function getExamples() { + return array( + 'api.php?action=query&prop=extracts&exchars=175&titles=Therion' => 'Get a 175-character extract', + ); + } + + + public function getHelpUrls() { + return 'https://www.mediawiki.org/wiki/Extension:MobileFrontend#prop.3Dextracts'; + } +} diff --git a/ExtractFormatter.php b/ExtractFormatter.php new file mode 100644 index 0000000..1508a7f --- /dev/null +++ b/ExtractFormatter.php @@ -0,0 +1,74 @@ +<?php + +/** + * Provides text-only or limited-HTML extracts of page HTML + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + */ +class ExtractFormatter extends HtmlFormatter { + const SECTION_MARKER_START = "\1\2"; + const SECTION_MARKER_END = "\2\1"; + + private $plainText; + + /** + * @param string $text: Text to convert + * @param bool $plainText: Whether extract should be plaintext + */ + public function __construct( $text, $plainText ) { + global $wgExtractsRemoveClasses; + + wfProfileIn( __METHOD__ ); + parent::__construct( HtmlFormatter::wrapHTML( $text ) ); + $this->plainText = $plainText; + + $this->setRemoveMedia( true ); + $this->remove( $wgExtractsRemoveClasses ); + + if ( $plainText ) { + $this->flattenAllTags(); + } else { + $this->flatten( array( 'span', 'a' ) ); + } + wfProfileOut( __METHOD__ ); + } + + public function getText( $dummy = null ) { + wfProfileIn( __METHOD__ ); + $this->filterContent(); + $text = parent::getText(); + if ( $this->plainText ) { + $text = html_entity_decode( $text ); + $text = str_replace( "\xC2\xA0", ' ', $text ); // replace nbsp with space + $text = str_replace( "\r", "\n", $text ); // for Windows + $text = preg_replace( "/\n{3,}/", "\n\n", $text ); // normalise newlines + } + wfProfileOut( __METHOD__ ); + return $text; + } + + public function onHtmlReady( $html ) { + wfProfileIn( __METHOD__ ); + if ( $this->plainText ) { + $html = preg_replace( '/\s*(<h([1-6])\b)/i', + "\n\n" . self::SECTION_MARKER_START . '$2' . self::SECTION_MARKER_END . '$1' , + $html + ); + } + wfProfileOut( __METHOD__ ); + return $html; + } +} diff --git a/ExtractFormatterTest.php b/ExtractFormatterTest.php new file mode 100644 index 0000000..c5a1b3d --- /dev/null +++ b/ExtractFormatterTest.php @@ -0,0 +1,38 @@ +<?php + +/** + * @group TextExtracts + * @group Broken + * Disabled for now due to Jenkins weirdness + */ +class ExtractFormatterTest extends MediaWikiTestCase { + /** + * @dataProvider provideExtracts + */ + public function testExtracts( $expected, $wikiText, $plainText ) { + $title = Title::newFromText( 'Test' ); + $po = new ParserOptions(); + $po->setEditSection( true ); + $parser = new Parser(); + $text = $parser->parse( $wikiText, $title, $po )->getText(); + $fmt = new ExtractFormatter( $text, $plainText ); + $fmt->remove( '.metadata' ); // Will be added via $wgExtractsRemoveClasses on WMF + $text = trim( $fmt->getText() ); + $this->assertEquals( $expected, $text ); + } + + public function provideExtracts() { + $dutch = "'''Dutch''' (<span class=\"unicode haudio\" style=\"white-space:nowrap;\"><span class=\"fn\">" + . "[[File:Loudspeaker.svg|11px|link=File:nl-Nederlands.ogg|About this sound]] [[:Media:nl-Nederlands.ogg|''Nederlands'']]" + . "</span> <small class=\"metadata audiolinkinfo\" style=\"cursor:help;\">([[Wikipedia:Media help|<span style=\"cursor:help;\">" + . "help</span>]]·[[:File:nl-Nederlands.ogg|<span style=\"cursor:help;\">info</span>]])</small></span>) is a" + . " [[West Germanic languages|West Germanic language]] and the native language of most of the population of the [[Netherlands]]"; + return array( + array( + "Dutch ( Nederlands ) is a West Germanic language and the native language of most of the population of the Netherlands", + $dutch, + true, + ), + ); + } +} \ No newline at end of file diff --git a/TextExtracts.i18n.php b/TextExtracts.i18n.php new file mode 100644 index 0000000..3cc9efe --- /dev/null +++ b/TextExtracts.i18n.php @@ -0,0 +1,20 @@ +<?php +/** + * Internationalisation for TextExtracts + * + * @file + * @ingroup Extensions + */ +$messages = array(); + +/** English + */ +$messages['en'] = array( + 'textextracts-desc' => 'Provides plain-text or limited HTML extracts of page content', +); + +/** Message documentation (Message documentation) + */ +$messages['qqq'] = array( + 'textextracts-desc' => '{{desc}}', +); \ No newline at end of file diff --git a/TextExtracts.php b/TextExtracts.php new file mode 100644 index 0000000..cc0cf0a --- /dev/null +++ b/TextExtracts.php @@ -0,0 +1,58 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +$wgExtensionCredits['other'][] = array( + 'path' => __FILE__, + 'name' => 'TextExtracts', + 'author' => array( 'Max Semenik' ), + 'descriptionmsg' => 'textextracts-desc', + 'url' => 'https://www.mediawiki.org/wiki/Extension:TextExtracts', +); + +define( 'TEXT_EXTRACTS_INSTALLED', true ); + +$dir = __DIR__; +$wgExtensionMessagesFiles['TextExtracts'] = "$dir/TextExtracts.i18n.php"; + + +$wgAutoloadClasses['ExtractFormatter'] = "$dir/ExtractFormatter.php"; +$wgAutoloadClasses['ApiQueryExtracts'] = "$dir/ApiQueryExtracts.php"; +$wgAPIPropModules['extracts'] = 'ApiQueryExtracts'; +$wgHooks['OpenSearchXml'][] = 'ApiQueryExtracts::onOpenSearchXml'; +$wgHooks['UnitTestsList'][] = function( &$files ) { + $files[] = __DIR__ . '/ExtractFormatterTest.php'; + return true; +}; + + +// Configuration variables + +/** + * Selectors of content to be removed from HTML + */ +$wgExtractsRemoveClasses = array( + '.toc', 'table', 'div', '.mw-editsection', 'sup.reference', 'span.coordinates', + 'span.geo-multi-punct', 'span.geo-nondefault', '.noexcerpt', '.error', '.nomobile', +); + +/** + * Whether this extension should provide its extracts for Extension:OpenSearchXml + */ +$wgExtractsExtendOpenSearchXml = false; -- To view, visit https://gerrit.wikimedia.org/r/102305 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I8a02f38554f6f06a00e6229889f7545f8ee8976a Gerrit-PatchSet: 2 Gerrit-Project: mediawiki/extensions/TextExtracts Gerrit-Branch: master Gerrit-Owner: MaxSem <[email protected]> Gerrit-Reviewer: Awjrichards <[email protected]> Gerrit-Reviewer: Kaldari <[email protected]> Gerrit-Reviewer: MaxSem <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
