MaxSem has uploaded a new change for review.
https://gerrit.wikimedia.org/r/102305
Change subject: Initial conversion of prop=extracts from MobileFrontend
......................................................................
Initial conversion of prop=extracts from MobileFrontend
Change-Id: I8a02f38554f6f06a00e6229889f7545f8ee8976a
---
A ApiQueryExtracts.php
A ExtractFormatter.php
A ExtractFormatterTest.php
A TextExtracts.i18n.php
A TextExtracts.php
5 files changed, 555 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/TextExtracts
refs/changes/05/102305/1
diff --git a/ApiQueryExtracts.php b/ApiQueryExtracts.php
new file mode 100644
index 0000000..11b9542
--- /dev/null
+++ b/ApiQueryExtracts.php
@@ -0,0 +1,395 @@
+<?php
+
+class ApiQueryExtracts extends ApiQueryBase {
+ /**
+ * @var ParserOptions
+ */
+ private $parserOptions;
+ private $params;
+
+ public function __construct( $query, $moduleName ) {
+ parent::__construct( $query, $moduleName, 'ex' );
+ }
+
+ public function execute() {
+ wfProfileIn( __METHOD__ );
+ $titles = $this->getPageSet()->getGoodTitles();
+ if ( count( $titles ) == 0 ) {
+ wfProfileOut( __METHOD__ );
+ return;
+ }
+ $isXml = $this->getMain()->isInternalMode() ||
$this->getMain()->getPrinter()->getFormat() == 'XML';
+ $result = $this->getResult();
+ $params = $this->params = $this->extractRequestParams();
+ $this->requireMaxOneParameter( $params, 'chars', 'sentences' );
+ $continue = 0;
+ $limit = intval( $params['limit'] );
+ if ( $limit > 1 && !$params['intro'] ) {
+ $limit = 1;
+ $result->setWarning( "exlimit was too large for a whole
article extracts request, lowered to $limit" );
+ }
+ if ( isset( $params['continue'] ) ) {
+ $continue = intval( $params['continue'] );
+ if ( $continue < 0 || $continue > count( $titles ) ) {
+ $this->dieUsageMsg( '_badcontinue' );
+ }
+ $titles = array_slice( $titles, $continue, null, true );
+ }
+ $count = 0;
+ /** @var Title $t */
+ foreach ( $titles as $id => $t ) {
+ if ( ++$count > $limit ) {
+ $this->setContinueEnumParameter( 'continue',
$continue + $count - 1 );
+ break;
+ }
+ $text = $this->getExtract( $t );
+ $pageName = $t->getText();
+ $text = $this->truncate( $text, $pageName );
+ if ( $this->params['plaintext'] ) {
+ $text = $this->doSections( $text );
+ }
+
+ if ( $isXml ) {
+ $fit = $result->addValue( array( 'query',
'pages', $id ), 'extract', array( '*' => $text ) );
+ } else {
+ $fit = $result->addValue( array( 'query',
'pages', $id ), 'extract', $text );
+ }
+ if ( !$fit ) {
+ $this->setContinueEnumParameter( 'continue',
$continue + $count - 1 );
+ break;
+ }
+ }
+ wfProfileOut( __METHOD__ );
+ }
+
+ /**
+ * OpenSearchXml hook handler
+ * @param array $results
+ * @return bool
+ */
+ public static function onOpenSearchXml( &$results ) {
+ global $wgExtractsExtendOpenSearchXml;
+ if ( !$wgExtractsExtendOpenSearchXml || !count( $results ) ) {
+ return true;
+ }
+ $pageIds = array_keys( $results );
+ $api = new ApiMain( new FauxRequest(
+ array(
+ 'action' => 'query',
+ 'prop' => 'extracts',
+ 'explaintext' => true,
+ 'exintro' => true,
+ 'exlimit' => count( $results ),
+ 'pageids' => implode( '|', $pageIds ),
+ ) )
+ );
+ $api->execute();
+ $data = $api->getResultData();
+ foreach ( $pageIds as $id ) {
+ if ( isset(
$data['query']['pages'][$id]['extract']['*'] ) ) {
+ $results[$id]['extract'] =
$data['query']['pages'][$id]['extract']['*'];
+ $results[$id]['extract trimmed'] = false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns a processed, but not trimmed extract
+ * @param Title $title
+ * @return string
+ */
+ private function getExtract( Title $title ) {
+ wfProfileIn( __METHOD__ );
+ $page = WikiPage::factory( $title );
+
+ $introOnly = $this->params['intro'];
+ $text = $this->getFromCache( $page, $introOnly );
+ // if we need just first section, try retrieving full page and
getting first section out of it
+ if ( $text === false && $introOnly ) {
+ $text = $this->getFromCache( $page, false );
+ if ( $text !== false ) {
+ $text = $this->getFirstSection( $text,
$this->params['plaintext'] );
+ }
+ }
+ if ( $text === false ) {
+ $text = $this->parse( $page );
+ $text = $this->convertText( $text, $title,
$this->params['plaintext'] );
+ $this->setCache( $page, $text );
+ }
+ wfProfileOut( __METHOD__ );
+ return $text;
+ }
+
+ private function cacheKey( WikiPage $page, $introOnly ) {
+ return wfMemcKey( 'mf', 'extract', $page->getLatest(),
$this->params['plaintext'], $introOnly );
+ }
+
+ private function getFromCache( WikiPage $page, $introOnly ) {
+ global $wgMemc;
+
+ $key = $this->cacheKey( $page, $introOnly );
+ return $wgMemc->get( $key );
+ }
+
+ private function setCache( WikiPage $page, $text ) {
+ global $wgMemc;
+
+ $key = $this->cacheKey( $page, $this->params['intro'] );
+ $wgMemc->set( $key, $text );
+ }
+
+ private function getFirstSection( $text, $plainText ) {
+ if ( $plainText ) {
+ $regexp = '/^(.*?)(?=' .
ExtractFormatter::SECTION_MARKER_START . ')/s';
+ } else {
+ $regexp = '/^(.*?)(?=<h[1-6]\b)/s';
+ }
+ if ( preg_match( $regexp, $text, $matches ) ) {
+ $text = $matches[0];
+ }
+ return $text;
+ }
+
+ /**
+ * Returns page HTML
+ * @param WikiPage $page
+ * @return string
+ */
+ private function parse( WikiPage $page ) {
+ wfProfileIn( __METHOD__ );
+ if ( !$this->parserOptions ) {
+ $this->parserOptions = new ParserOptions( new User(
'127.0.0.1' ) );
+ }
+ // first try finding full page in parser cache
+ if ( $page->isParserCacheUsed( $this->parserOptions, 0 ) ) {
+ $pout = ParserCache::singleton()->get( $page,
$this->parserOptions );
+ if ( $pout ) {
+ $text = $pout->getText();
+ if ( $this->params['intro'] ) {
+ $text = $this->getFirstSection( $text,
false );
+ }
+ wfProfileOut( __METHOD__ );
+ return $text;
+ }
+ }
+ $request = array(
+ 'action' => 'parse',
+ 'page' => $page->getTitle()->getPrefixedText(),
+ 'prop' => 'text'
+ );
+ if ( $this->params['intro'] ) {
+ $request['section'] = 0;
+ }
+ // in case of cache miss, render just the needed section
+ $api = new ApiMain( new FauxRequest( $request ) );
+ $api->execute();
+ $data = $api->getResultData();
+ wfProfileOut( __METHOD__ );
+ return $data['parse']['text']['*'];
+ }
+
+ /**
+ * Converts page HTML into an extract
+ * @param string $text
+ * @return string
+ */
+ private function convertText( $text ) {
+ wfProfileIn( __METHOD__ );
+ $fmt = new ExtractFormatter( $text, $this->params['plaintext'],
$this->params['sectionformat'] );
+ $text = $fmt->getText();
+
+ wfProfileOut( __METHOD__ );
+ return trim( $text );
+ }
+
+ /**
+ * Truncate the given text to a certain number of characters or
sentences
+ * @param string $text The text to truncate
+ * @param string $pageName Title of the page (for debugging)
+ * @return string
+ */
+ private function truncate( $text, $pageName ) {
+ if ( $this->params['chars'] ) {
+ return $this->getFirstChars( $text,
$this->params['chars'] );
+ } elseif ( $this->params['sentences'] ) {
+ return $this->getFirstSentences( $text,
$this->params['sentences'], $pageName );
+ }
+ return $text;
+ }
+
+ /**
+ *
+ * @param string $text
+ * @param int $requestedLength
+ * @return string
+ */
+ private function getFirstChars( $text, $requestedLength ) {
+ wfProfileIn( __METHOD__ );
+ $length = mb_strlen( $text );
+ if ( $length <= $requestedLength ) {
+ wfProfileOut( __METHOD__ );
+ return $text;
+ }
+ $pattern = "#^.{{$requestedLength}}[\\w/]*>?#su";
+ preg_match( $pattern, $text, $m );
+ $text = $m[0];
+ // Fix possibly unclosed tags
+ $text = $this->tidy( $text );
+ $text .= wfMessage( 'ellipsis' )->inContentLanguage()->text();
+ wfProfileOut( __METHOD__ );
+ return $text;
+ }
+
+ /**
+ *
+ * @param string $text
+ * @param int $requestedSentenceCount
+ * @param string $pageName Title of the page (for debugging)
+ * @return string
+ */
+ private function getFirstSentences( $text, $requestedSentenceCount,
$pageName ) {
+ wfProfileIn( __METHOD__ );
+ // Based on code from OpenSearchXml by Brion Vibber
+ $endchars = array(
+ '([^\d])\.\s', '\!\s', '\?\s', // regular ASCII
+ '。', // full-width ideographic full-stop
+ '．', '！', '？', // double-width roman forms
+ '｡', // half-width ideographic full stop
+ );
+
+ $endgroup = implode( '|', $endchars );
+ $end = "(?:$endgroup)";
+ $sentence = ".+?$end+";
+ $regexp = "/^($sentence){1,{$requestedSentenceCount}}/u";
+ $matches = array();
+ $res = preg_match( $regexp, $text, $matches );
+ if( $res ) {
+ $text = $matches[0];
+ } else {
+ if ( $res === false ) {
+ wfLogWarning( "Regular expresssion compilation
failure. Page: $pageName; RegEx: $regexp" );
+ }
+ // Just return the first line
+ $lines = explode( "\n", $text );
+ $text = trim( $lines[0] );
+ }
+ $text = $this->tidy( $text );
+ wfProfileOut( __METHOD__ );
+ return $text;
+ }
+
+ /**
+ * A simple wrapper around tidy
+ * @param string $text
+ * @return string
+ */
+ private function tidy( $text ) {
+ global $wgUseTidy;
+
+ wfProfileIn( __METHOD__ );
+ if ( $wgUseTidy && !$this->params['plaintext'] ) {
+ $text = trim ( MWTidy::tidy( $text ) );
+ }
+ wfProfileOut( __METHOD__ );
+ return $text;
+ }
+
+ private function doSections( $text ) {
+ $text = preg_replace_callback(
+ "/" . ExtractFormatter::SECTION_MARKER_START . '(\d)'.
ExtractFormatter::SECTION_MARKER_END . "(.*?)$/m",
+ array( $this, 'sectionCallback' ),
+ $text
+ );
+ return $text;
+ }
+
+ private function sectionCallback( $matches ) {
+ if ( $this->params['sectionformat'] == 'raw' ) {
+ return $matches[0];
+ }
+ $func = __CLASS__ .
"::doSection_{$this->params['sectionformat']}";
+ return call_user_func( $func, $matches[1], trim( $matches[2] )
);
+ }
+
+ private static function doSection_wiki( $level, $text ) {
+ $bars = str_repeat( '=', $level );
+ return "\n$bars $text $bars";
+ }
+
+ private static function doSection_plain( $level, $text ) {
+ return "\n$text";
+ }
+
+ public function getAllowedParams() {
+ return array(
+ 'chars' => array(
+ ApiBase::PARAM_TYPE => 'integer',
+ ApiBase::PARAM_MIN => 1,
+ ),
+ 'sentences' => array(
+ ApiBase::PARAM_TYPE => 'integer',
+ ApiBase::PARAM_MIN => 1,
+ ApiBase::PARAM_MAX => 10,
+ ),
+ 'limit' => array(
+ ApiBase::PARAM_DFLT => 1,
+ ApiBase::PARAM_TYPE => 'limit',
+ ApiBase::PARAM_MIN => 1,
+ ApiBase::PARAM_MAX => 20,
+ ApiBase::PARAM_MAX2 => 20,
+ ),
+ 'intro' => false,
+ 'plaintext' => false,
+ 'sectionformat' => array(
+ ApiBase::PARAM_TYPE => array( 'plain', 'wiki',
'raw' ),
+ ApiBase::PARAM_DFLT => 'wiki',
+ ),
+ 'continue' => array(
+ ApiBase::PARAM_TYPE => 'integer',
+ ),
+ );
+ }
+
+ public function getParamDescription() {
+ return array(
+ 'chars' => 'How many characters to return, actual text
returned might be slightly longer.',
+ 'sentences' => 'How many sentences to return',
+ 'limit' => 'How many extracts to return. ',
+ 'intro' => 'Return only content before the first
section',
+ 'plaintext' => 'Return extracts as plaintext instead of
limited HTML',
+ 'sectionformat' => array(
+ 'How to format sections in plaintext mode:',
+ ' plain - No formatting',
+ ' wiki - Wikitext-style formatting == like this
==',
+ " raw - This module's internal representation
(secton titles prefixed with <ASCII 1><ASCII 2><section level><ASCII 2><ASCII
1>",
+ ),
+ 'continue' => 'When more results are available, use
this to continue',
+ );
+ }
+
+ public function getDescription() {
+ return 'Returns plain-text or limited HTML extracts of the
given page(s)';
+ }
+
+ public function getPossibleErrors() {
+ return array_merge( parent::getPossibleErrors(), array(
+ array( 'code' => '_badcontinue', 'info' => 'Invalid
continue param. You should pass the original value returned by the previous
query' ),
+ ) );
+ }
+
+ public function getExamples() {
+ return array(
+
'api.php?action=query&prop=extracts&exchars=175&titles=Therion' => 'Get a
175-character extract',
+ );
+ }
+
+
+ public function getHelpUrls() {
+ return
'https://www.mediawiki.org/wiki/Extension:MobileFrontend#prop.3Dextracts';
+ }
+
+ public function getVersion() {
+ return __CLASS__ . ': $Id$';
+ }
+}
diff --git a/ExtractFormatter.php b/ExtractFormatter.php
new file mode 100644
index 0000000..f68b552
--- /dev/null
+++ b/ExtractFormatter.php
@@ -0,0 +1,59 @@
+<?php
+
+/**
+ * Provides text-only or limited-HTML extracts of page HTML
+ */
+class ExtractFormatter extends HtmlFormatter {
+ const SECTION_MARKER_START = "\1\2";
+ const SECTION_MARKER_END = "\2\1";
+
+ private $plainText;
+
+ /**
+ * @param string $text: Text to convert
+ * @param bool $plainText: Whether extract should be plaintext
+ */
+ public function __construct( $text, $plainText ) {
+ global $wgExtractsRemoveClasses;
+
+ wfProfileIn( __METHOD__ );
+ parent::__construct( HtmlFormatter::wrapHTML( $text ) );
+ $this->plainText = $plainText;
+
+ $this->setRemoveMedia( true );
+ $this->remove( $wgExtractsRemoveClasses );
+
+ if ( $plainText ) {
+ $this->flattenAllTags();
+ } else {
+ $this->flatten( array( 'span', 'a' ) );
+ }
+ wfProfileOut( __METHOD__ );
+ }
+
+ public function getText( $dummy = null ) {
+ wfProfileIn( __METHOD__ );
+ $this->filterContent();
+ $text = parent::getText();
+ if ( $this->plainText ) {
+ $text = html_entity_decode( $text );
+ $text = str_replace( "\xC2\xA0", ' ', $text ); //
replace nbsp with space
+ $text = str_replace( "\r", "\n", $text ); // for Windows
+ $text = preg_replace( "/\n{3,}/", "\n\n", $text ); //
normalise newlines
+ }
+ wfProfileOut( __METHOD__ );
+ return $text;
+ }
+
+ public function onHtmlReady( $html ) {
+ wfProfileIn( __METHOD__ );
+ if ( $this->plainText ) {
+ $html = preg_replace( '/\s*(<h([1-6])\b)/i',
+ "\n\n" . self::SECTION_MARKER_START . '$2' .
self::SECTION_MARKER_END . '$1' ,
+ $html
+ );
+ }
+ wfProfileOut( __METHOD__ );
+ return $html;
+ }
+}
diff --git a/ExtractFormatterTest.php b/ExtractFormatterTest.php
new file mode 100644
index 0000000..c5a1b3d
--- /dev/null
+++ b/ExtractFormatterTest.php
@@ -0,0 +1,38 @@
+<?php
+
+/**
+ * @group TextExtracts
+ * @group Broken
+ * Disabled for now due to Jenkins weirdness
+ */
+class ExtractFormatterTest extends MediaWikiTestCase {
+ /**
+ * @dataProvider provideExtracts
+ */
+ public function testExtracts( $expected, $wikiText, $plainText ) {
+ $title = Title::newFromText( 'Test' );
+ $po = new ParserOptions();
+ $po->setEditSection( true );
+ $parser = new Parser();
+ $text = $parser->parse( $wikiText, $title, $po )->getText();
+ $fmt = new ExtractFormatter( $text, $plainText );
+ $fmt->remove( '.metadata' ); // Will be added via
$wgExtractsRemoveClasses on WMF
+ $text = trim( $fmt->getText() );
+ $this->assertEquals( $expected, $text );
+ }
+
+ public function provideExtracts() {
+ $dutch = "'''Dutch''' (<span class=\"unicode haudio\"
style=\"white-space:nowrap;\"><span class=\"fn\">"
+ .
"[[File:Loudspeaker.svg|11px|link=File:nl-Nederlands.ogg|About this
sound]] [[:Media:nl-Nederlands.ogg|''Nederlands'']]"
+ . "</span> <small class=\"metadata audiolinkinfo\"
style=\"cursor:help;\">([[Wikipedia:Media help|<span style=\"cursor:help;\">"
+ . "help</span>]]·[[:File:nl-Nederlands.ogg|<span
style=\"cursor:help;\">info</span>]])</small></span>) is a"
+ . " [[West Germanic languages|West Germanic language]]
and the native language of most of the population of the [[Netherlands]]";
+ return array(
+ array(
+ "Dutch ( Nederlands ) is a West Germanic
language and the native language of most of the population of the Netherlands",
+ $dutch,
+ true,
+ ),
+ );
+ }
+}
\ No newline at end of file
diff --git a/TextExtracts.i18n.php b/TextExtracts.i18n.php
new file mode 100644
index 0000000..3cc9efe
--- /dev/null
+++ b/TextExtracts.i18n.php
@@ -0,0 +1,20 @@
+<?php
+/**
+ * Internationalisation for TextExtracts
+ *
+ * @file
+ * @ingroup Extensions
+ */
+$messages = array();
+
+/** English
+ */
+$messages['en'] = array(
+ 'textextracts-desc' => 'Provides plain-text or limited HTML extracts of
page content',
+);
+
+/** Message documentation (Message documentation)
+ */
+$messages['qqq'] = array(
+ 'textextracts-desc' => '{{desc}}',
+);
\ No newline at end of file
diff --git a/TextExtracts.php b/TextExtracts.php
new file mode 100644
index 0000000..f02bfc0
--- /dev/null
+++ b/TextExtracts.php
@@ -0,0 +1,43 @@
+<?php
+/**
+ *
+ */
+
+$wgExtensionCredits['other'][] = array(
+ 'path' => __FILE__,
+ 'name' => 'TextExtracts',
+ 'author' => array( 'Max Semenik' ),
+ 'descriptionmsg' => 'textextracts-desc',
+ 'url' => 'https://www.mediawiki.org/wiki/Extension:TextExtracts',
+);
+
+define( 'TEXT_EXTRACTS_INSTALLED', true );
+
+$dir = __DIR__;
+$wgExtensionMessagesFiles['TextExtracts'] = "$dir/TextExtracts.i18n.php";
+
+
+$wgAutoloadClasses['ExtractFormatter'] = "$dir/ExtractFormatter.php";
+$wgAutoloadClasses['ApiQueryExtracts'] = "$dir/ApiQueryExtracts.php";
+$wgAPIPropModules['extracts'] = 'ApiQueryExtracts';
+$wgHooks['OpenSearchXml'][] = 'ApiQueryExtracts::onOpenSearchXml';
+$wgHooks['UnitTestsList'][] = function( &$files ) {
+ $files[] = __DIR__ . '/ExtractFormatterTest.php';
+ return true;
+};
+
+
+// Configuration variables
+
+/**
+ * Selectors of content to be removed from HTML
+ */
+$wgExtractsRemoveClasses = array(
+ '.toc', 'table', 'div', '.mw-editsection', 'sup.reference',
'span.coordinates',
+ 'span.geo-multi-punct', 'span.geo-nondefault', '.noexcerpt', '.error',
'.nomobile',
+);
+
+/**
+ * Whether this extension should provide its extracts for
Extension:OpenSearchXml
+ */
+$wgExtractsExtendOpenSearchXml = false;
--
To view, visit https://gerrit.wikimedia.org/r/102305
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8a02f38554f6f06a00e6229889f7545f8ee8976a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/TextExtracts
Gerrit-Branch: master
Gerrit-Owner: MaxSem <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
