This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "FusionForge".
The branch, master has been updated
via acc792cbd36773a1e2f2f8fd42e09d4e951f1511 (commit)
via 333e7db5eb1aa9ad10d47a55726cba996fe91405 (commit)
from d0416f20595f246ae41eb8012bfcdd5437941401 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
https://scm.fusionforge.org/anonscm/gitweb/?p=fusionforge/fusionforge.git;a=commitdiff;h=acc792cbd36773a1e2f2f8fd42e09d4e951f1511
commit acc792cbd36773a1e2f2f8fd42e09d4e951f1511
Author: Franck Villaume <[email protected]>
Date: Sun Dec 13 15:01:10 2015 +0100
docman parser: simplify usage, support UTF-8
diff --git a/src/common/docman/Document.class.php
b/src/common/docman/Document.class.php
index 67540bd..76277cd 100644
--- a/src/common/docman/Document.class.php
+++ b/src/common/docman/Document.class.php
@@ -186,7 +186,7 @@ class Document extends Error {
// key words for in-document search
if ($this->Group->useDocmanSearch() && $filesize) {
$kw = new Parsedata();
- $kwords = $kw->get_parse_data($data,
htmlspecialchars($title), htmlspecialchars($description), $filetype, $filename);
+ $kwords = $kw->get_parse_data($data, $filetype);
} else {
$kwords ='';
}
@@ -836,7 +836,7 @@ class Document extends Error {
// key words for in-document search
if ($this->Group->useDocmanSearch()) {
$kw = new Parsedata();
- $kwords = $kw->get_parse_data($data,
htmlspecialchars($title), htmlspecialchars($description), $filetype, $filename);
+ $kwords = $kw->get_parse_data($data, $filetype);
} else {
$kwords = '';
}
@@ -999,8 +999,8 @@ class Document extends Error {
*/
function downloadUp() {
if (session_loggedin()) {
- $s =& session_get_user();
- $us = $s->getID();
+ global $LUSER;
+ $us = $LUSER->getID();
} else {
$us=100;
}
diff --git a/src/common/docman/Parsedata.class.php
b/src/common/docman/Parsedata.class.php
index 2fcbcc1..a0b1b5a 100644
--- a/src/common/docman/Parsedata.class.php
+++ b/src/common/docman/Parsedata.class.php
@@ -4,7 +4,7 @@
*
* Copyright 2005, Fabio Bertagnin
* Copyright 2009-2010, Franck Villaume - Capgemini
- * Copyright 2011-2013, Franck Villaume - TrivialDev
+ * Copyright 2011-2013,2015 Franck Villaume - TrivialDev
* Copyright (C) 2011-2012 Alain Peyrat - Alcatel-Lucent
* http://fusionforge.org
*
@@ -43,14 +43,11 @@ class Parsedata {
/**
* get_parse_data - analyse content and metadata
*
- * @param string $data the path of the file to be
analysed
- * @param string $title the file title
- * @param string $description the file description
- * @param string $filetype the file type
- * @param string $filename the filename
+ * @param string $data the path of the file to analyse
+ * @param string $filetype type of the file to analyse
* @return string the analysed content
*/
- function get_parse_data($data, $title, $description, $filetype,
$filename) {
+ function get_parse_data($data, $filetype) {
$parser = "";
$rep = "";
if (array_key_exists($filetype, $this->parsers)) {
@@ -59,21 +56,8 @@ class Parsedata {
$cmd = "php -f $parser $data";
$rep = shell_exec($cmd);
}
- // always parse title, description, filename and filetype
- $data1 = utf8_decode("$title $description $filename $filetype");
- // temporary file for treatment
- $filename = tempnam(sys_get_temp_dir(), 'docman');
- $handle = fopen($filename, 'w');
- fwrite($handle, $data1);
- fclose($handle);
- $cmd = $this->p_path.$this->parsers['text/plain'];
- $cmd = "php -f $cmd $filename";
- $rep1 = shell_exec($cmd);
- if ( file_exists ($filename ) ) {
- unlink($filename);
- }
// dont need to unlink the filename because parser_text already
remove it
- return preg_replace("/\n/", " ", "$rep $rep1");
+ return preg_replace("/\n/", " ", "$rep");
}
/**
diff --git a/src/common/docman/engine/parser_text.inc.php
b/src/common/docman/engine/parser_text.inc.php
index e821884..c2c878a 100644
--- a/src/common/docman/engine/parser_text.inc.php
+++ b/src/common/docman/engine/parser_text.inc.php
@@ -5,7 +5,7 @@
* Copyright 2005, Fabio Bertagnin
* Copyright 2011, Franck Villaume - Capgemini
* Copyright (C) 2012 Alain Peyrat - Alcatel-Lucent
- * Copyright 2012, Franck Villaume - TrivialDev
+ * Copyright 2012,2015 Franck Villaume - TrivialDev
* http://fusionforge.org
*
* This file is part of FusionForge. FusionForge is free software;
@@ -43,14 +43,13 @@ function parser_text($fichin) {
$buff = strtolower($buff);
}
- // élimination d'éventuels caractères unicode encore présents
+ // transform into UTF-8
if (function_exists('mb_convert_encoding')) {
- $buff = mb_convert_encoding($buff, 'ascii');
+ $buff = mb_convert_encoding($buff, 'UTF-8');
}
- // élimination caractères avec accents
- // et caractères spéciaux
- $buff = suppression_diacritics($buff);
+ // remove specific characters
+ $buff = delete_specific_chars($buff);
// tous les mots dans un tableau
$words = explode(' ', $buff);
// élimination des doublons
@@ -68,9 +67,8 @@ function print_list($list) {
return $rep;
}
-function suppression_diacritics($text) {
- $text = iconv('UTF-8', 'US-ASCII//TRANSLIT', $text) ;
- $text = strtr($text, "\t\r\n?.*'\":;,#![]()", " ");
+function delete_specific_chars($text) {
+ $text = strtr($text, "\t\r\n?.*'\":;,#![]()/", " ");
return $text;
}
diff --git a/src/cronjobs/db/update_docdata_dataword.php
b/src/cronjobs/db/update_docdata_dataword.php
index 55c11d6..5c47898 100755
--- a/src/cronjobs/db/update_docdata_dataword.php
+++ b/src/cronjobs/db/update_docdata_dataword.php
@@ -6,7 +6,7 @@
* Copyright 2005, Fabio Bertagnin
* Copyright 2009, Roland Mas
* Copyright 2010-2011, Franck Villaume - Capgemini
- * Copyright 2012, Franck Villaume - TrivialDev
+ * Copyright 2012,2015 Franck Villaume - TrivialDev
*
* This file is part of FusionForge. FusionForge is free software;
* you can redistribute it and/or modify it under the terms of the
@@ -36,7 +36,7 @@ $p = new Parsedata();
$timestarttrait = microtime_float();
// documents list
$resarr = array();
-$result = db_query_params('SELECT doc_data.docid, doc_data.group_id,
doc_data.filename, doc_data.title, doc_data.description, doc_data.filetype from
doc_data, groups where doc_data.group_id = groups.group_id and
groups.force_docman_reindex = $1',
+$result = db_query_params('SELECT doc_data.docid, doc_data.group_id,
doc_data.filename, doc_data.filetype from doc_data, groups where
doc_data.group_id = groups.group_id and groups.force_docman_reindex = $1',
array('1'));
if (!$result) {
die(db_error());
@@ -48,6 +48,8 @@ if ($result) {
}
}
+db_free_result($result);
+
$compt = 0;
$errorFlag = 0;
foreach ($resarr as $item) {
@@ -60,13 +62,12 @@ foreach ($resarr as $item) {
fwrite($fh, $d->getFileData(false));
fclose($fh);
$lenin = $d->getFileSize();
- $res = $p->get_parse_data($datafile, htmlspecialchars($item['title']),
htmlspecialchars($item['description']), $item['filetype'], $item['filename']);
+ $res = $p->get_parse_data($datafile, $item['filetype']);
$len = strlen($res);
if (file_exists($datafile)) {
unlink($datafile);
}
- $resUp = db_query_params('UPDATE doc_data SET data_words=$1 WHERE
docid=$2',
- array ($res, $item['docid']));
+ $resUp = db_query_params('UPDATE doc_data SET data_words=$1 WHERE
docid=$2', array ($res, $item['docid']));
if (!$resUp) {
die('unable to update data: '.db_error());
}
https://scm.fusionforge.org/anonscm/gitweb/?p=fusionforge/fusionforge.git;a=commitdiff;h=333e7db5eb1aa9ad10d47a55726cba996fe91405
commit 333e7db5eb1aa9ad10d47a55726cba996fe91405
Author: Franck Villaume <[email protected]>
Date: Sun Dec 13 14:59:57 2015 +0100
fix doc_data update_vectors: wrong table update
diff --git a/src/db/20151212-docman-fix-update_vectors.sql
b/src/db/20151212-docman-fix-update_vectors.sql
new file mode 100644
index 0000000..9b47619
--- /dev/null
+++ b/src/db/20151212-docman-fix-update_vectors.sql
@@ -0,0 +1,119 @@
+CREATE OR REPLACE FUNCTION update_vectors() RETURNS TRIGGER AS '
+DECLARE
+table_name TEXT;
+BEGIN
+ table_name := TG_ARGV[0];
+ -- **** artifact table ****
+ IF table_name = ''artifact'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM artifact_idx WHERE
artifact_id=OLD.artifact_id;
+ ELSE
+ DELETE FROM artifact_idx WHERE
artifact_id=NEW.artifact_id;
+ INSERT INTO artifact_idx (SELECT a.artifact_id,
to_tsvector(a.artifact_id::text) || to_tsvector(a.summary) ||
to_tsvector(a.details) || coalesce(ff_tsvector_agg(to_tsvector(am.body)),
to_tsvector('''')) AS vectors FROM artifact a LEFT OUTER JOIN artifact_message
am USING (artifact_id) WHERE a.artifact_id=NEW.artifact_id GROUP BY
a.artifact_id, a.summary, a.details);
+ END IF;
+ -- **** artifact_message table ****
+ ELSIF table_name = ''artifact_message'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM artifact_idx WHERE
artifact_id=OLD.artifact_id;
+ ELSE
+ DELETE FROM artifact_idx WHERE
artifact_id=NEW.artifact_id;
+ INSERT INTO artifact_idx (SELECT a.artifact_id,
to_tsvector(a.artifact_id::text) || to_tsvector(a.summary) ||
to_tsvector(a.details) || coalesce(ff_tsvector_agg(to_tsvector(am.body)),
to_tsvector('''')) AS vectors FROM artifact a LEFT OUTER JOIN artifact_message
am USING (artifact_id) WHERE a.artifact_id=NEW.artifact_id GROUP BY
a.artifact_id, a.summary, a.details);
+ END IF;
+ -- **** doc_data table ****
+ ELSIF table_name = ''doc_data'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO doc_data_idx (docid, group_id, vectors)
VALUES (NEW.docid, NEW.group_id, to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.description,'''')));
+ INSERT INTO doc_data_words_idx (docid, group_id,
vectors) VALUES (NEW.docid, NEW.group_id, to_tsvector(coalesce(NEW.title,'''')
||'' ''|| coalesce(NEW.description,'''') ||'' ''|| coalesce(NEW.filename,'''')
||'' ''|| coalesce(NEW.filetype,'''') ||'' ''|| coalesce(NEW.data_words,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE doc_data_idx SET group_id = NEW.group_id,
vectors=to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.description,'''')) WHERE docid=NEW.docid;
+ UPDATE doc_data_words_idx SET group_id = NEW.group_id,
vectors=to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.description,'''') ||'' ''|| coalesce(NEW.filename,'''') ||'' ''||
coalesce(NEW.filetype,'''') ||'' ''|| coalesce(NEW.data_words,'''')) WHERE
docid=NEW.docid;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM doc_data_idx WHERE docid=OLD.docid;
+ DELETE FROM doc_data_words_idx WHERE docid=OLD.docid;
+ END IF;
+ -- **** forum table ****
+ ELSIF table_name = ''forum'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO forum_idx (msg_id, group_id, vectors)
(SELECT f.msg_id, g.group_id, to_tsvector(coalesce(f.subject,'''') ||'' ''||
+ coalesce(f.body,'''')) AS vectors FROM forum f,
forum_group_list g WHERE f.group_forum_id = g.group_forum_id AND f.msg_id =
NEW.msg_id);
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE forum_idx SET
vectors=to_tsvector(coalesce(NEW.subject,'''') ||'' ''||
coalesce(NEW.body,'''')) WHERE msg_id=NEW.msg_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM forum_idx WHERE msg_id=OLD.msg_id;
+ END IF;
+ -- **** frs_file table ****
+ ELSIF table_name = ''frs_file'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO frs_file_idx (file_id, release_id, vectors)
VALUES (NEW.file_id, NEW.release_id, to_tsvector(coalesce(NEW.filename,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE frs_file_idx SET
vectors=to_tsvector(coalesce(NEW.filename,'''')), release_id=NEW.release_id
WHERE file_id=NEW.file_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM frs_file_idx WHERE file_id=OLD.file_id;
+ END IF;
+ -- **** frs_release table ****
+ ELSIF table_name = ''frs_release'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO frs_release_idx (release_id, vectors)
VALUES (NEW.release_id, to_tsvector(coalesce(NEW.changes,'''') ||'' ''||
coalesce(NEW.notes,'''') ||'' ''|| coalesce(NEW.name,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE frs_release_idx SET
vectors=to_tsvector(coalesce(NEW.changes,'''') ||'' ''||
coalesce(NEW.notes,'''') ||'' ''|| coalesce(NEW.name,'''')) WHERE
release_id=NEW.release_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM frs_release_idx WHERE
release_id=OLD.release_id;
+ DELETE FROM frs_file_idx WHERE
release_id=OLD.release_id;
+ END IF;
+ -- **** groups table ****
+ ELSIF table_name = ''groups'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO groups_idx (group_id, vectors) VALUES
(NEW.group_id, to_tsvector(coalesce(NEW.group_name,'''') ||'' ''||
coalesce(NEW.short_description,'''') ||'' ''||
coalesce(NEW.unix_group_name,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE groups_idx SET
vectors=to_tsvector(coalesce(NEW.group_name,'''') ||'' ''||
coalesce(NEW.short_description,'''') ||'' ''||
coalesce(NEW.unix_group_name,'''')) WHERE group_id=NEW.group_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM groups_idx WHERE group_id=OLD.group_id;
+ END IF;
+ -- **** news_bytes table ****
+ ELSIF table_name = ''news_bytes'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO news_bytes_idx (id, vectors) VALUES
(NEW.id, to_tsvector(coalesce(NEW.summary,'''') ||'' ''||
coalesce(NEW.details,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE news_bytes_idx SET
vectors=to_tsvector(coalesce(NEW.summary,'''') ||'' ''||
coalesce(NEW.details,'''')) WHERE id=NEW.id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM news_bytes_idx WHERE id=OLD.id;
+ END IF;
+ -- **** project_task table ****
+ ELSIF table_name = ''project_task'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM project_task_idx WHERE
project_task_id=OLD.project_task_id;
+ ELSE
+ DELETE FROM project_task_idx WHERE
project_task_id=NEW.project_task_id;
+ INSERT INTO project_task_idx (SELECT t.project_task_id,
to_tsvector(t.project_task_id::text) || to_tsvector(t.summary) ||
to_tsvector(t.details) || coalesce(ff_tsvector_agg(to_tsvector(tm.body)),
to_tsvector('''')) AS vectors FROM project_task t LEFT OUTER JOIN
project_messages tm USING (project_task_id) WHERE
t.project_task_id=NEW.project_task_id GROUP BY t.project_task_id, t.summary,
t.details);
+ END IF;
+ -- **** project_messages table ****
+ ELSIF table_name = ''project_messages'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM project_task_idx WHERE
project_task_id=OLD.project_task_id;
+ ELSE
+ DELETE FROM project_task_idx WHERE
project_task_id=NEW.project_task_id;
+ INSERT INTO project_task_idx (SELECT t.project_task_id,
to_tsvector(t.summary) || to_tsvector(t.details) ||
coalesce(ff_tsvector_agg(to_tsvector(tm.body)), to_tsvector('''')) AS vectors
FROM project_task t LEFT OUTER JOIN project_messages tm USING (project_task_id)
WHERE t.project_task_id=NEW.project_task_id GROUP BY t.project_task_id,
t.summary, t.details);
+ END IF;
+ -- **** skills_data table ****
+ ELSIF table_name = ''skills_data'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO skills_data_idx (skills_data_id, vectors)
VALUES (NEW.skills_data_id, to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.keywords,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE skills_data_idx SET
vectors=to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.keywords,'''')) WHERE skills_data_id=NEW.skills_data_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM skills_data_idx WHERE
skills_data_id=OLD.skills_data_id;
+ END IF;
+ -- **** users table ****
+ ELSIF table_name = ''users'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO users_idx (user_id, vectors) VALUES
(NEW.user_id, to_tsvector(coalesce(NEW.user_name,'''') ||'' ''||
coalesce(NEW.realname,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE users_idx SET
vectors=to_tsvector(coalesce(NEW.user_name,'''') ||'' ''||
coalesce(NEW.realname,'''')) WHERE user_id=NEW.user_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM users_idx WHERE user_id=OLD.user_id;
+ END IF;
+ END IF;
+
+ RETURN NEW;
+END;'
+LANGUAGE 'plpgsql';
-----------------------------------------------------------------------
Summary of changes:
src/common/docman/Document.class.php | 8 +++----
src/common/docman/Parsedata.class.php | 26 +++++-----------------
src/common/docman/engine/parser_text.inc.php | 16 ++++++-------
src/cronjobs/db/update_docdata_dataword.php | 11 ++++-----
....sql => 20151212-docman-fix-update_vectors.sql} | 5 ++++-
5 files changed, 26 insertions(+), 40 deletions(-)
copy src/db/{20140304-fix_update_vectors_function.sql =>
20151212-docman-fix-update_vectors.sql} (89%)
hooks/post-receive
--
FusionForge
_______________________________________________
Fusionforge-commits mailing list
[email protected]
http://lists.fusionforge.org/cgi-bin/mailman/listinfo/fusionforge-commits
