[MediaWiki-commits] [Gerrit] Connect to db2 and import the math table there - change (mediawiki...MathSearch)
Physikerwelt has uploaded a new change for review.
https://gerrit.wikimedia.org/r/104916
Change subject: Connect to db2 and import the math table there
..
Connect to db2 and import the math table there
Change-Id: I064622eb875db2259405cf1868a3256e3637aa72
---
M maintenance/ExportMathTable.php
M maintenance/IndexBase.php
M maintenance/UpdateMath.php
A maintenance/db2test.php
4 files changed, 71 insertions(+), 27 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/MathSearch
refs/changes/16/104916/1
diff --git a/maintenance/ExportMathTable.php b/maintenance/ExportMathTable.php
index ca01daf..7338740 100644
--- a/maintenance/ExportMathTable.php
+++ b/maintenance/ExportMathTable.php
@@ -28,12 +28,16 @@
*
*/
class ExportMathTable extends IndexBase {
+ private $db2Pass;
+ private $statment;
/**
*
*/
public function __construct() {
parent::__construct();
$this-mDescription = 'Exports a db2 compatible math index
table.';
+ $this-addArg( 'passw', If set, the data is directly imported
to db2, false );
+ $this-addArg( 'truncate', If true, db2 math table is deleted
before import, false );
}
/**
@@ -44,9 +48,12 @@
$mo = MathObject::constructformpagerow($row);
$out = ''. $mo-getMd5().'';
$out .= ','. $mo-getTex().'';
- $out .= ','. $row-mathindex_page_id .'';
- $out .= ','. $row-mathindex_anchor.'';
- $out .= ','.str_replace(array('',\n),array('',' '),
$mo-getMathml()).'';
+ $out .= ','. $row-mathindex_page_id .'';
+ $out .= ','. $row-mathindex_anchor.'';
+ $out .= ','.str_replace(array('',\n),array('',' '),
$mo-getMathml()).'';
+ if( $this-db2Pass ) {
+
$this-statment-execute(array($mo-getMd5(),$mo-getTex(),$row-mathindex_page_id,$row-mathindex_anchor,$mo-getMathml()));
+}
return $out.\n;
}
@@ -55,6 +62,22 @@
*
*/
public function execute() {
+ $this-db2Pass = $this-getOption( 'passw',false );
+ if ( $this-db2Pass ){
+ try {
+ $connection = new PDO(ibm:MATH, db2inst1,
$this-db2Pass, array(
+ PDO::ATTR_PERSISTENT = TRUE,
+ PDO::ATTR_ERRMODE =
PDO::ERRMODE_EXCEPTION)
+ );
+ } catch (Exception $e) {
+ echo($e-getMessage());
+ }
+ if ( $this-getOption('truncate' , false ) ){
+$connection-query('TRUNCATE TABLE
wiki.math IMMEDIATE');
+ }
+ $this-statment = $connection-prepare('insert into
wiki.math (math_md5, math_tex, mathindex_pageid, mathindex_anchord,
math_mathml) values(?, ?, ?, ?, ?)');
+
+ }
parent::execute();
}
}
diff --git a/maintenance/IndexBase.php b/maintenance/IndexBase.php
index 4958703..5edfe95 100644
--- a/maintenance/IndexBase.php
+++ b/maintenance/IndexBase.php
@@ -38,6 +38,7 @@
$this-mDescription = 'Exports data';
$this-addArg( 'dir', 'The directory where the harvest files go
to.' );
$this-addArg( 'ffmax', The maximal number of formula per
file., false );
+ $this-addOption( 'limit', 'The maximal number of database
entries to be considered', false );
}
/**
@@ -86,8 +87,11 @@
array( 'mathindex', 'mathoid' ),
array( 'mathindex_page_id', 'mathindex_anchor',
'math_mathml', 'math_inputhash', 'mathindex_inputhash' ),// $vars
(columns of the table)
'math_inputhash = mathindex_inputhash'
- , __METHOD__,
- array( 'ORDER BY' = 'mathindex_page_id' ) );
+ , __METHOD__
+ ,array(
+ 'LIMIT'= $this-getOption(
'limit',1 ) ,
+ 'ORDER BY' = 'mathindex_page_id' )
+ );
echo write . $this-res-numRows() . results to index\n;
do {
$fn = $this-getArg( 0 ) . '/math' . sprintf( '%012d',
$i ) . '.xml';
diff --git a/maintenance/UpdateMath.php b/maintenance/UpdateMath.php
index c3b2c7c..bfb2c61 100644
--- a/maintenance/UpdateMath.php
+++ b/maintenance/UpdateMath.php
@@ -22,7 +22,7 @@
require_once( dirname( __FILE__ ) . '/../../../maintenance/Maintenance.php' );
class UpdateMath extends Maintenance {
- const RTI_CHUNK_SIZE = 500;
+
[MediaWiki-commits] [Gerrit] Connect to db2 and import the math table there - change (mediawiki...MathSearch)
Physikerwelt has submitted this change and it was merged.
Change subject: Connect to db2 and import the math table there
..
Connect to db2 and import the math table there
Change-Id: I064622eb875db2259405cf1868a3256e3637aa72
---
M maintenance/ExportMathTable.php
M maintenance/IndexBase.php
M maintenance/UpdateMath.php
A maintenance/db2test.php
4 files changed, 71 insertions(+), 27 deletions(-)
Approvals:
Physikerwelt: Verified; Looks good to me, approved
diff --git a/maintenance/ExportMathTable.php b/maintenance/ExportMathTable.php
index ca01daf..7338740 100644
--- a/maintenance/ExportMathTable.php
+++ b/maintenance/ExportMathTable.php
@@ -28,12 +28,16 @@
*
*/
class ExportMathTable extends IndexBase {
+ private $db2Pass;
+ private $statment;
/**
*
*/
public function __construct() {
parent::__construct();
$this-mDescription = 'Exports a db2 compatible math index
table.';
+ $this-addArg( 'passw', If set, the data is directly imported
to db2, false );
+ $this-addArg( 'truncate', If true, db2 math table is deleted
before import, false );
}
/**
@@ -44,9 +48,12 @@
$mo = MathObject::constructformpagerow($row);
$out = ''. $mo-getMd5().'';
$out .= ','. $mo-getTex().'';
- $out .= ','. $row-mathindex_page_id .'';
- $out .= ','. $row-mathindex_anchor.'';
- $out .= ','.str_replace(array('',\n),array('',' '),
$mo-getMathml()).'';
+ $out .= ','. $row-mathindex_page_id .'';
+ $out .= ','. $row-mathindex_anchor.'';
+ $out .= ','.str_replace(array('',\n),array('',' '),
$mo-getMathml()).'';
+ if( $this-db2Pass ) {
+
$this-statment-execute(array($mo-getMd5(),$mo-getTex(),$row-mathindex_page_id,$row-mathindex_anchor,$mo-getMathml()));
+}
return $out.\n;
}
@@ -55,6 +62,22 @@
*
*/
public function execute() {
+ $this-db2Pass = $this-getOption( 'passw',false );
+ if ( $this-db2Pass ){
+ try {
+ $connection = new PDO(ibm:MATH, db2inst1,
$this-db2Pass, array(
+ PDO::ATTR_PERSISTENT = TRUE,
+ PDO::ATTR_ERRMODE =
PDO::ERRMODE_EXCEPTION)
+ );
+ } catch (Exception $e) {
+ echo($e-getMessage());
+ }
+ if ( $this-getOption('truncate' , false ) ){
+$connection-query('TRUNCATE TABLE
wiki.math IMMEDIATE');
+ }
+ $this-statment = $connection-prepare('insert into
wiki.math (math_md5, math_tex, mathindex_pageid, mathindex_anchord,
math_mathml) values(?, ?, ?, ?, ?)');
+
+ }
parent::execute();
}
}
diff --git a/maintenance/IndexBase.php b/maintenance/IndexBase.php
index 4958703..5edfe95 100644
--- a/maintenance/IndexBase.php
+++ b/maintenance/IndexBase.php
@@ -38,6 +38,7 @@
$this-mDescription = 'Exports data';
$this-addArg( 'dir', 'The directory where the harvest files go
to.' );
$this-addArg( 'ffmax', The maximal number of formula per
file., false );
+ $this-addOption( 'limit', 'The maximal number of database
entries to be considered', false );
}
/**
@@ -86,8 +87,11 @@
array( 'mathindex', 'mathoid' ),
array( 'mathindex_page_id', 'mathindex_anchor',
'math_mathml', 'math_inputhash', 'mathindex_inputhash' ),// $vars
(columns of the table)
'math_inputhash = mathindex_inputhash'
- , __METHOD__,
- array( 'ORDER BY' = 'mathindex_page_id' ) );
+ , __METHOD__
+ ,array(
+ 'LIMIT'= $this-getOption(
'limit',1 ) ,
+ 'ORDER BY' = 'mathindex_page_id' )
+ );
echo write . $this-res-numRows() . results to index\n;
do {
$fn = $this-getArg( 0 ) . '/math' . sprintf( '%012d',
$i ) . '.xml';
diff --git a/maintenance/UpdateMath.php b/maintenance/UpdateMath.php
index c3b2c7c..bfb2c61 100644
--- a/maintenance/UpdateMath.php
+++ b/maintenance/UpdateMath.php
@@ -22,7 +22,7 @@
require_once( dirname( __FILE__ ) . '/../../../maintenance/Maintenance.php' );
class UpdateMath extends Maintenance {
- const RTI_CHUNK_SIZE = 500;
+ const RTI_CHUNK_SIZE = 100;
var $purge = false;
