Soeren.oldag has uploaded a new change for review.
https://gerrit.wikimedia.org/r/205841
Change subject: Minor tweaks in special pages and UpdateTable script.
......................................................................
Minor tweaks in special pages and UpdateTable script.
Added id column to ExternalDbs special page. Multiple external values are now
split in several lines in CrossCheck special page. Improved output of
UpdateTable script.
Change-Id: Id03e3a4044976d6be6411cdb8f8a01159a828123
---
M i18n/en.json
M i18n/qqq.json
M includes/UpdateTable/Importer.php
M specials/SpecialCrossCheck.php
M specials/SpecialExternalDbs.php
5 files changed, 67 insertions(+), 37 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/WikidataQualityExternalValidation
refs/changes/41/205841/1
diff --git a/i18n/en.json b/i18n/en.json
index 457debe..98dcdae 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -8,13 +8,14 @@
"wikidataquality-crosscheck-instructions-example": "Try for example Q159099
(Nina Hagen) and look at the results.",
"wikidataquality-crosscheck-empty-result": "The entered entity has no
statements, that can be cross-checked with any supported database.",
"wikidataquality-crosscheck-result-table-header-local-value": "Wikidata
value",
- "wikidataquality-crosscheck-result-table-header-external-value": "External
value",
+ "wikidataquality-crosscheck-result-table-header-external-value": "External
values",
"wikidataquality-crosscheck-result-table-header-external-source": "External
source",
"wikidataquality-externaldbs": "List of external databases",
"wikidataquality-externaldbs-instructions": "All databases that are
supported for cross-checks against Wikidata entites are listed below.",
"wikidataquality-externaldbs-overview-headline": "Databases",
"wikidataquality-externaldbs-name": "Name",
+ "wikidataquality-externaldbs-id": "Id",
"wikidataquality-externaldbs-import-date": "Import date",
"wikidataquality-externaldbs-language": "Data language",
"wikidataquality-externaldbs-source-urls": "Source URLs",
diff --git a/i18n/qqq.json b/i18n/qqq.json
index 048cdb1..5f51ece 100644
--- a/i18n/qqq.json
+++ b/i18n/qqq.json
@@ -18,6 +18,7 @@
"wikidataquality-externaldbs-instructions": "Purpose of
SpecialPage:ExternalDbs",
"wikidataquality-externaldbs-overview-headline": "Headline that appears
above the overview of external databases.\n{{Identical|Database}}",
"wikidataquality-externaldbs-name": "Name of the external
source.\n{{Identical|Name}}",
+ "wikidataquality-externaldbs-id": "Id of the dump of a external source.",
"wikidataquality-externaldbs-import-date": "Name for import date from
the external source.",
"wikidataquality-externaldbs-language": "Name for language from the
external source.",
"wikidataquality-externaldbs-source-urls": "Name for source urls from
the external source.",
diff --git a/includes/UpdateTable/Importer.php
b/includes/UpdateTable/Importer.php
index 07d6003..56b9704 100644
--- a/includes/UpdateTable/Importer.php
+++ b/includes/UpdateTable/Importer.php
@@ -59,23 +59,27 @@
$this->extractTarFile();
$db = $this->establishDbConnection();
-
if ( !$this->assertTablesExist( $db ) ) {
return;
}
$dumpIds = $this->insertMetaInformation( $db );
- $this->deleteOldDatabaseEntries( $db, $dumpIds, DUMP_DATA_TABLE );
+ if ( !$this->importContext->isQuiet() ) {
+ print "\n";
+ }
+ $this->deleteOldDatabaseEntries( $db, $dumpIds, DUMP_DATA_TABLE );
$this->deleteOldDatabaseEntries( $db, $dumpIds,
DUMP_IDENTIFIER_PROPERTIES_TABLE );
- $this->insertIdentifierProperties( $db );
+ if ( !$this->importContext->isQuiet() ) {
+ print "\n";
+ }
+ $this->insertIdentifierProperties( $db );
$this->insertExternalValues( $db );
$this->deleteExtractedFiles();
-
$this->reuseDbConnection( $db );
}
@@ -123,10 +127,7 @@
$tablesUsed = array( DUMP_META_TABLE,
DUMP_IDENTIFIER_PROPERTIES_TABLE, DUMP_DATA_TABLE );
foreach ( $tablesUsed as $tableName ) {
if ( !$db->tableExists( $tableName ) ) {
- if ( !$this->importContext->isQuiet() ) {
- print "$tableName table does not exist.\nExecuting
core/maintenance/update.php may help.\n";
- }
-
+ print "$tableName table does not exist.\nExecuting
core/maintenance/update.php may help.\n";
return false;
}
}
@@ -168,8 +169,7 @@
global $wgDBtype;
if ( !$this->importContext->isQuiet() ) {
- print "Removing old entries\n";
- print "\n";
+ print "Removing old entries from $tableName\n";
}
foreach ( $dumpIds as $dumpId ) {
@@ -190,11 +190,11 @@
print "$i batches deleted";
}
} while ( $db->affectedRows() > 0 );
-
- if ( !$this->importContext->isQuiet() ) {
- print "\n";
- }
}
+ }
+
+ if ( !$this->importContext->isQuiet() ) {
+ print "\n";
}
}
@@ -206,8 +206,13 @@
* @return array
*/
protected function insertMetaInformation( DatabaseBase $db ) {
+ if ( !$this->importContext->isQuiet() ) {
+ print "Insert new dump meta information\n";
+ }
+
$csvFile = fopen( $this->dumpInformationFilePath, 'rb' );
+ $i = 0;
$dumpIds = array();
while ( $data = fgetcsv( $csvFile ) ) {
$metaInformation = new DumpMetaInformation(
@@ -222,9 +227,19 @@
$metaInformation->save( $db );
$dumpIds[ ] = $metaInformation->getDumpId();
+
+ $i++;
+ if ( !$this->importContext->isQuiet() ) {
+ print "\r\033[K";
+ print "$i rows inserted or updated";
+ }
}
fclose( $csvFile );
+
+ if ( !$this->importContext->isQuiet() ) {
+ print "\n";
+ }
return $dumpIds;
}
@@ -235,32 +250,43 @@
* @param DatabaseBase $db
*/
protected function insertIdentifierProperties( DatabaseBase $db ) {
+ if ( !$this->importContext->isQuiet() ) {
+ print "Insert new identifier properties\n";
+ }
+
$csvFile = fopen( $this->identifierPropertiesFilePath, 'rb' );
- while ( $data = fgetcsv( $csvFile ) ) {
- $accumulator = array(
+ $i = 0;
+ $accumulator = array();
+ while ( true ) {
+ $data = fgetcsv( $csvFile );
+ if ( $data === false || ++$i %
$this->importContext->getBatchSize() === 0 ) {
+ $db->commit( __METHOD__, 'flush' );
+ wfWaitForSlaves();
+ $db->insert( DUMP_IDENTIFIER_PROPERTIES_TABLE, $accumulator );
+ if ( !$this->importContext->isQuiet() ) {
+ print "\r\033[K";
+ print "$i rows inserted";
+ }
+
+ $accumulator = array();
+
+ if ( $data === false ) {
+ break;
+ }
+ }
+
+ $accumulator[ ] = array(
'identifier_pid' => $data[ 0 ],
'dump_id' => $data[ 1 ]
);
-
- $existing = $db->selectRow(
- DUMP_IDENTIFIER_PROPERTIES_TABLE,
- array(
- 'identifier_pid',
- 'dump_id'
- ),
- $accumulator
- );
-
- if ( !$existing ) {
- $db->insert(
- DUMP_IDENTIFIER_PROPERTIES_TABLE,
- $accumulator
- );
- }
}
fclose( $csvFile );
+
+ if ( !$this->importContext->isQuiet() ) {
+ print "\n";
+ }
}
/**
@@ -270,7 +296,7 @@
*/
protected function insertExternalValues( DatabaseBase $db ) {
if ( !$this->importContext->isQuiet() ) {
- print "Insert new entries\n";
+ print "Insert new data values\n";
}
$csvFile = fopen( $this->externalValuesFilePath, 'rb' );
@@ -303,10 +329,10 @@
);
}
+ fclose( $csvFile );
+
if ( !$this->importContext->isQuiet() ) {
print "\n";
}
-
- fclose( $csvFile );
}
}
\ No newline at end of file
diff --git a/specials/SpecialCrossCheck.php b/specials/SpecialCrossCheck.php
index 4056e83..1a945c1 100755
--- a/specials/SpecialCrossCheck.php
+++ b/specials/SpecialCrossCheck.php
@@ -111,7 +111,7 @@
$this->formatStatus(
$result->getStatus() ),
$this->entityIdHtmlLinkFormatter->formatEntityId( $result->getPropertyId() ),
$this->formatDataValues(
$result->getCompareResult()->getLocalValue() ),
- $this->formatDataValues(
$result->getCompareResult()->getExternalValues() ),
+ $this->formatDataValues(
$result->getCompareResult()->getExternalValues(), true, Html::element('br') ),
$this->entityIdHtmlLinkFormatter->formatEntityId(
$result->getDumpMetaInformation()->getSourceItemId() )
)
);
diff --git a/specials/SpecialExternalDbs.php b/specials/SpecialExternalDbs.php
index 431008b..ce455f6 100644
--- a/specials/SpecialExternalDbs.php
+++ b/specials/SpecialExternalDbs.php
@@ -64,6 +64,7 @@
$table = new HtmlTable(
array (
$this->msg(
'wikidataquality-externaldbs-name' )->text(),
+ $this->msg( 'wikidataquality-externaldbs-id' )->text(),
$this->msg(
'wikidataquality-externaldbs-import-date' )->text(),
$this->msg(
'wikidataquality-externaldbs-language' )->text(),
$this->msg(
'wikidataquality-externaldbs-source-urls' )->text(),
@@ -99,6 +100,7 @@
foreach ( $dumpMetaInformationGroup as $dumpMetaInformation ) {
$rows[ ] = array (
+ $dumpMetaInformation->getDumpId(),
$this->formatDateTime(
$dumpMetaInformation->getImportDate() ),
Language::fetchLanguageName(
$dumpMetaInformation->getLanguage(),
--
To view, visit https://gerrit.wikimedia.org/r/205841
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id03e3a4044976d6be6411cdb8f8a01159a828123
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/WikidataQualityExternalValidation
Gerrit-Branch: master
Gerrit-Owner: Soeren.oldag <soeren_ol...@freenet.de>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
