This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "FusionForge".
The branch, master has been updated
via b3ccf27fa637966f4d2f13f639a609234607e016 (commit)
via 8f0a85734cf5498136ad41188457497e3872332e (commit)
via 914f995dcb144b75515579c2fb354f4d09af2146 (commit)
via 0bc6911b5e2fb16ee18bad17152a6512a879c6a6 (commit)
via 9da2f9a96504d1ae4033cb63a565492861404cc8 (commit)
from 75dbc48d80d7c70131251649246886d55418816f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
https://scm.fusionforge.org/anonscm/gitweb/?p=fusionforge/fusionforge.git;a=commitdiff;h=b3ccf27fa637966f4d2f13f639a609234607e016
commit b3ccf27fa637966f4d2f13f639a609234607e016
Author: Roland Mas <[email protected]>
Date: Tue May 3 20:06:23 2016 +0200
Document latest changes to search
diff --git a/src/CHANGES b/src/CHANGES
index 2725edb..9fcd59d 100644
--- a/src/CHANGES
+++ b/src/CHANGES
@@ -15,6 +15,8 @@ FusionForge 6.X:
* Plugin AuthBuiltin: add captcha after 3 attempts with the same login [#795]
(TrivialDev)
* Widget MySystasks: new widget for user to display systasks perform on user
projects (TrivialDev)
* Widget ProjectScmStats: new widget for project to display SCM stats
(TrivialDev)
+* Search: index project tags and use them for search (Roland Mas)
+* Search: provide language-specific settings for better indexation/search
(Roland Mas)
FusionForge 6.0.4:
* Account Management System: fix sysCreateUser for UNIX: unix_gid dropped
(TrivialDev)
https://scm.fusionforge.org/anonscm/gitweb/?p=fusionforge/fusionforge.git;a=commitdiff;h=8f0a85734cf5498136ad41188457497e3872332e
commit 8f0a85734cf5498136ad41188457497e3872332e
Author: Roland Mas <[email protected]>
Date: Tue May 3 13:54:38 2016 +0200
Testsuite for search improvements (language support + project tags)
diff --git a/tests/func/30_Search/searchTest.php
b/tests/func/30_Search/searchTest.php
index 06d5758..abbd0ee 100644
--- a/tests/func/30_Search/searchTest.php
+++ b/tests/func/30_Search/searchTest.php
@@ -648,5 +648,64 @@ class Search extends FForge_SeleniumTestCase
$this->assertFalse($this->isTextPresent("Doc2"));
$this->assertTrue($this->isTextPresent("News1"));
$this->assertTrue($this->isTextPresent("News2"));
- }
+
+ // Test language-specific search configurations
+ // Also test project search based on tags
+ $this->createAndGoto('projectc');
+ $this->click("link=Admin");
+ $this->type("//input[@name='form_shortdesc'","Coding hôtels
Iiİı");
+ $this->type("//input[@name='form_tags'","Goldorak");
+ $this->clickAndWait("//input[@name='submit']");
+
+ // In "simple" configuration, no normalization occurs beyond
capitalization
+
$this->runCommand(dirname(__FILE__).'/../../src/bin/configure-fti-search.php
simple');
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "coded");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertFalse($this->isTextPresent("projectc"));
+
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "goldorak");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertTrue($this->isTextPresent("projectc"));
+
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "hotel");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertFalse($this->isTextPresent("projectc"));
+
+ // In English, accents are removed and coding==coded
+
$this->runCommand(dirname(__FILE__).'/../../src/bin/configure-fti-search.php
english');
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "coded");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertTrue($this->isTextPresent("projectc"));
+
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "goldorak");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertTrue($this->isTextPresent("projectc"));
+
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "hotel");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertTrue($this->isTextPresent("projectc"));
+
+ // In French, accents are removed but coding==coded
+
$this->runCommand(dirname(__FILE__).'/../../src/bin/configure-fti-search.php
french');
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "coded");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertFalse($this->isTextPresent("projectc"));
+
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "goldorak");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertTrue($this->isTextPresent("projectc"));
+
+ $this->open(ROOT) ;
+ $this->type("//input[@name='words']", "hotel");
+ $this->clickAndWait("//input[@name='Search']");
+ $this->assertTrue($this->isTextPresent("projectc"));
+}
}
https://scm.fusionforge.org/anonscm/gitweb/?p=fusionforge/fusionforge.git;a=commitdiff;h=914f995dcb144b75515579c2fb354f4d09af2146
commit 914f995dcb144b75515579c2fb354f4d09af2146
Author: Roland Mas <[email protected]>
Date: Tue May 3 11:52:01 2016 +0200
Added script to configure text search
diff --git a/src/bin/configure-fti-search.php b/src/bin/configure-fti-search.php
new file mode 100755
index 0000000..5ee10cb
--- /dev/null
+++ b/src/bin/configure-fti-search.php
@@ -0,0 +1,102 @@
+#! /usr/bin/php -f
+<?php
+/**
+ * FusionForge source control management
+ *
+ * Copyright 2016, Roland Mas
+ *
+ * This file is part of FusionForge. FusionForge is free software;
+ * you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software
+ * Foundation; either version 2 of the Licence, or (at your option)
+ * any later version.
+ *
+ * FusionForge is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with FusionForge; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+require (dirname(__FILE__).'/../common/include/env.inc.php');
+require_once $gfcommon.'include/pre.php';
+
+$known_configs = array(
+ 'simple' => array(
+ 'clone' => 'simple',
+ 'queries' => array()
+ ),
+ );
+
+$langs = array(
+ 'danish',
+ 'dutch',
+ 'english',
+ 'finnish',
+ 'french',
+ 'german',
+ 'hungarian',
+ 'italian',
+ 'norwegian',
+ 'portuguese',
+ 'romanian',
+ 'russian',
+ 'spanish',
+ 'swedish',
+ 'turkish',
+ );
+
+foreach ($langs as $l) {
+ $known_configs[$l] = array(
+ 'clone' => $l,
+ 'queries' => array(
+ 'ALTER TEXT SEARCH CONFIGURATION fusionforge ALTER
MAPPING FOR hword, hword_part, word WITH unaccent, '.$l.'_stem',
+ )
+ );
+}
+
+function usage() {
+ global $known_configs;
+ $l = array_keys($known_configs);
+ sort($l);
+ echo "Usage: .../configure-fti-search.php <configuration>
+Currently implemented configurations: ".implode(' ',$l)."\n";
+ exit(1);
+}
+
+if (count($argv) != 2) {
+ usage();
+}
+$chosen = $argv[1];
+if (!array_key_exists($chosen,$known_configs)) {
+ usage();
+}
+$config = $known_configs[$chosen];
+
+function query_and_exit_if_error($q,$p=array()) {
+ $res = db_query_params($q,$p);
+ if (!$res) {
+ db_rollback();
+ print db_error();
+ exit(1);
+ }
+}
+
+db_begin();
+query_and_exit_if_error('DROP TEXT SEARCH CONFIGURATION fusionforge');
+query_and_exit_if_error('CREATE TEXT SEARCH CONFIGURATION fusionforge ( COPY =
'.$config['clone'].' )');
+foreach ($config['queries'] as $q) {
+ query_and_exit_if_error ($q);
+}
+query_and_exit_if_error('SELECT
to_tsvector($1,$2)',array('fusionforge','Hôtels camping forge Iıİi'));
+db_commit();
+
+system($fusionforge_basedir.'/bin/rebuild-fti-indices.php');
+
+// Local Variables:
+// mode: php
+// c-file-style: "bsd"
+// End:
diff --git a/src/db/20160503-configurable-fti-config.sql
b/src/db/20160503-configurable-fti-config.sql
new file mode 100644
index 0000000..e7c61c5
--- /dev/null
+++ b/src/db/20160503-configurable-fti-config.sql
@@ -0,0 +1,5 @@
+-- This only clones a text search configuration
+-- Its actual content is handled by a PHP script
+
+CREATE TEXT SEARCH CONFIGURATION fusionforge (COPY='simple');
+CREATE EXTENSION IF NOT EXISTS unaccent;
diff --git a/src/etc/config.ini.d/defaults.ini
b/src/etc/config.ini.d/defaults.ini
index c1818ab..90194fc 100644
--- a/src/etc/config.ini.d/defaults.ini
+++ b/src/etc/config.ini.d/defaults.ini
@@ -57,7 +57,7 @@ use_activity = yes
use_docman = yes
use_forum = yes
use_frs = yes
-fti_config = simple
+fti_config = fusionforge
use_ftp = yes
use_ftp_uploads = no
use_gateways = yes
https://scm.fusionforge.org/anonscm/gitweb/?p=fusionforge/fusionforge.git;a=commitdiff;h=0bc6911b5e2fb16ee18bad17152a6512a879c6a6
commit 0bc6911b5e2fb16ee18bad17152a6512a879c6a6
Author: Roland Mas <[email protected]>
Date: Tue May 3 16:34:34 2016 +0200
Use SQL function to rebuild indices
diff --git a/src/bin/rebuild-fti-indices.php b/src/bin/rebuild-fti-indices.php
index ec98c10..2f75456 100755
--- a/src/bin/rebuild-fti-indices.php
+++ b/src/bin/rebuild-fti-indices.php
@@ -21,22 +21,5 @@
require (dirname(__FILE__).'/../common/include/env.inc.php');
require_once $gfcommon."include/pre.php";
-$f = array(
- 'groups' => 'short_description',
- 'artifact' => 'summary',
- 'artifact_message' => 'body',
- 'doc_data' => 'title',
- 'forum' => 'subject',
- 'frs_file' => 'filename',
- 'frs_release' => 'name',
- 'news_bytes' => 'summary',
- 'project_task' => 'summary',
- 'project_messages' => 'body',
- 'skills_data' => 'keywords',
- 'users' => 'realname',
-);
-
-foreach ($f as $table => $column) {
- echo "Regenerating FTI indices for table $table.\n";
- $res = db_query_params ("UPDATE $table SET $column=$column");
-}
+echo "Regenerating all FTI indices.\n";
+$res = db_query_params ("SELECT rebuild_fti_indices()");
https://scm.fusionforge.org/anonscm/gitweb/?p=fusionforge/fusionforge.git;a=commitdiff;h=9da2f9a96504d1ae4033cb63a565492861404cc8
commit 9da2f9a96504d1ae4033cb63a565492861404cc8
Author: Roland Mas <[email protected]>
Date: Tue May 3 16:34:08 2016 +0200
Index project tags for full text search
diff --git a/src/db/20160502-fti-for-project-tags.sql
b/src/db/20160502-fti-for-project-tags.sql
new file mode 100644
index 0000000..fdbc7b2
--- /dev/null
+++ b/src/db/20160502-fti-for-project-tags.sql
@@ -0,0 +1,135 @@
+CREATE OR REPLACE FUNCTION update_vectors() RETURNS TRIGGER AS '
+DECLARE
+table_name TEXT;
+BEGIN
+ table_name := TG_ARGV[0];
+ -- **** artifact table ****
+ IF table_name = ''artifact'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM artifact_idx WHERE
artifact_id=OLD.artifact_id;
+ ELSE
+ DELETE FROM artifact_idx WHERE
artifact_id=NEW.artifact_id;
+ INSERT INTO artifact_idx (SELECT a.artifact_id,
to_tsvector(a.artifact_id::text) || to_tsvector(a.summary) ||
to_tsvector(a.details) || coalesce(ff_tsvector_agg(to_tsvector(am.body)),
to_tsvector('''')) AS vectors FROM artifact a LEFT OUTER JOIN artifact_message
am USING (artifact_id) WHERE a.artifact_id=NEW.artifact_id GROUP BY
a.artifact_id, a.summary, a.details);
+ END IF;
+ -- **** artifact_message table ****
+ ELSIF table_name = ''artifact_message'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM artifact_idx WHERE
artifact_id=OLD.artifact_id;
+ ELSE
+ DELETE FROM artifact_idx WHERE
artifact_id=NEW.artifact_id;
+ INSERT INTO artifact_idx (SELECT a.artifact_id,
to_tsvector(a.artifact_id::text) || to_tsvector(a.summary) ||
to_tsvector(a.details) || coalesce(ff_tsvector_agg(to_tsvector(am.body)),
to_tsvector('''')) AS vectors FROM artifact a LEFT OUTER JOIN artifact_message
am USING (artifact_id) WHERE a.artifact_id=NEW.artifact_id GROUP BY
a.artifact_id, a.summary, a.details);
+ END IF;
+ -- **** doc_data table ****
+ ELSIF table_name = ''doc_data'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO doc_data_idx (docid, group_id, vectors)
VALUES (NEW.docid, NEW.group_id, to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.description,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE doc_data_idx SET group_id=NEW.group_id,
vectors=to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.description,'''')) WHERE docid=NEW.docid;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM doc_data_idx WHERE docid=OLD.docid;
+ END IF;
+ -- **** forum table ****
+ ELSIF table_name = ''forum'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO forum_idx (msg_id, group_id, vectors)
(SELECT f.msg_id, g.group_id, to_tsvector(coalesce(f.subject,'''') ||'' ''||
+ coalesce(f.body,'''')) AS vectors FROM forum f,
forum_group_list g WHERE f.group_forum_id = g.group_forum_id AND f.msg_id =
NEW.msg_id);
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE forum_idx SET
vectors=to_tsvector(coalesce(NEW.subject,'''') ||'' ''||
coalesce(NEW.body,'''')) WHERE msg_id=NEW.msg_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM forum_idx WHERE msg_id=OLD.msg_id;
+ END IF;
+ -- **** frs_file table ****
+ ELSIF table_name = ''frs_file'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO frs_file_idx (file_id, release_id, vectors)
VALUES (NEW.file_id, NEW.release_id, to_tsvector(coalesce(NEW.filename,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE frs_file_idx SET
vectors=to_tsvector(coalesce(NEW.filename,'''')), release_id=NEW.release_id
WHERE file_id=NEW.file_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM frs_file_idx WHERE file_id=OLD.file_id;
+ END IF;
+ -- **** frs_release table ****
+ ELSIF table_name = ''frs_release'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO frs_release_idx (release_id, vectors)
VALUES (NEW.release_id, to_tsvector(coalesce(NEW.changes,'''') ||'' ''||
coalesce(NEW.notes,'''') ||'' ''|| coalesce(NEW.name,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE frs_release_idx SET
vectors=to_tsvector(coalesce(NEW.changes,'''') ||'' ''||
coalesce(NEW.notes,'''') ||'' ''|| coalesce(NEW.name,'''')) WHERE
release_id=NEW.release_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM frs_release_idx WHERE
release_id=OLD.release_id;
+ DELETE FROM frs_file_idx WHERE
release_id=OLD.release_id;
+ END IF;
+ -- **** groups table ****
+ ELSIF table_name = ''groups'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM groups_idx WHERE group_id=OLD.group_id;
+ ELSE
+ DELETE FROM groups_idx WHERE group_id=NEW.group_id;
+ INSERT INTO groups_idx (group_id, vectors) (SELECT
g.group_id, to_tsvector(coalesce(g.group_name,'''') ||'' ''||
coalesce(g.short_description,'''') ||'' ''||
coalesce(g.unix_group_name,'''')||'' ''||
coalesce(ff_tsvector_agg(to_tsvector(t.name)),to_tsvector(''''))) FROM groups g
LEFT OUTER JOIN project_tags t USING (group_id) WHERE g.group_id = NEW.group_id
GROUP BY g.group_id ORDER BY g.group_id);
+ END IF;
+ -- **** news_bytes table ****
+ ELSIF table_name = ''news_bytes'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO news_bytes_idx (id, vectors) VALUES
(NEW.id, to_tsvector(coalesce(NEW.summary,'''') ||'' ''||
coalesce(NEW.details,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE news_bytes_idx SET
vectors=to_tsvector(coalesce(NEW.summary,'''') ||'' ''||
coalesce(NEW.details,'''')) WHERE id=NEW.id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM news_bytes_idx WHERE id=OLD.id;
+ END IF;
+ -- **** project_task table ****
+ ELSIF table_name = ''project_task'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM project_task_idx WHERE
project_task_id=OLD.project_task_id;
+ ELSE
+ DELETE FROM project_task_idx WHERE
project_task_id=NEW.project_task_id;
+ INSERT INTO project_task_idx (SELECT t.project_task_id,
to_tsvector(t.project_task_id::text) || to_tsvector(t.summary) ||
to_tsvector(t.details) || coalesce(ff_tsvector_agg(to_tsvector(tm.body)),
to_tsvector('''')) AS vectors FROM project_task t LEFT OUTER JOIN
project_messages tm USING (project_task_id) WHERE
t.project_task_id=NEW.project_task_id GROUP BY t.project_task_id, t.summary,
t.details);
+ END IF;
+ -- **** project_messages table ****
+ ELSIF table_name = ''project_messages'' THEN
+ IF TG_OP = ''DELETE'' THEN
+ DELETE FROM project_task_idx WHERE
project_task_id=OLD.project_task_id;
+ ELSE
+ DELETE FROM project_task_idx WHERE
project_task_id=NEW.project_task_id;
+ INSERT INTO project_task_idx (SELECT t.project_task_id,
to_tsvector(t.summary) || to_tsvector(t.details) ||
coalesce(ff_tsvector_agg(to_tsvector(tm.body)), to_tsvector('''')) AS vectors
FROM project_task t LEFT OUTER JOIN project_messages tm USING (project_task_id)
WHERE t.project_task_id=NEW.project_task_id GROUP BY t.project_task_id,
t.summary, t.details);
+ END IF;
+ -- **** skills_data table ****
+ ELSIF table_name = ''skills_data'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO skills_data_idx (skills_data_id, vectors)
VALUES (NEW.skill_data_id, to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.keywords,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE skills_data_idx SET
vectors=to_tsvector(coalesce(NEW.title,'''') ||'' ''||
coalesce(NEW.keywords,'''')) WHERE skills_data_id=NEW.skills_data_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM skills_data_idx WHERE
skills_data_id=OLD.skills_data_id;
+ END IF;
+ -- **** users table ****
+ ELSIF table_name = ''users'' THEN
+ IF TG_OP = ''INSERT'' THEN
+ INSERT INTO users_idx (user_id, vectors) VALUES
(NEW.user_id, to_tsvector(coalesce(NEW.user_name,'''') ||'' ''||
coalesce(NEW.realname,'''')));
+ ELSIF TG_OP = ''UPDATE'' THEN
+ UPDATE users_idx SET
vectors=to_tsvector(coalesce(NEW.user_name,'''') ||'' ''||
coalesce(NEW.realname,'''')) WHERE user_id=NEW.user_id;
+ ELSIF TG_OP = ''DELETE'' THEN
+ DELETE FROM users_idx WHERE user_id=OLD.user_id;
+ END IF;
+ END IF;
+
+ RETURN NEW;
+END;'
+LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION rebuild_fti_indices() RETURNS void AS $$
+BEGIN
+ UPDATE groups SET short_description=short_description;
+ UPDATE artifact SET summary=summary;
+ UPDATE artifact_message SET body=body;
+ UPDATE doc_data SET title=title;
+ UPDATE forum SET subject=subject;
+ UPDATE frs_file SET filename=filename;
+ UPDATE frs_release SET name=name;
+ UPDATE news_bytes SET summary=summary;
+ UPDATE project_task SET summary=summary;
+ UPDATE project_messages SET body=body;
+ UPDATE skills_data SET keywords=keywords;
+ UPDATE users SET realname=realname;
+END;
+$$ LANGUAGE 'plpgsql';
+
+-- Rebuild all indices
+SELECT rebuild_fti_indices();
-----------------------------------------------------------------------
Summary of changes:
src/CHANGES | 2 +
src/bin/configure-fti-search.php | 102 +++++++++++++++++++++
src/bin/rebuild-fti-indices.php | 21 +----
...ction.sql => 20160502-fti-for-project-tags.sql} | 33 +++++--
src/db/20160503-configurable-fti-config.sql | 5 +
src/etc/config.ini.d/defaults.ini | 2 +-
tests/func/30_Search/searchTest.php | 61 +++++++++++-
7 files changed, 198 insertions(+), 28 deletions(-)
create mode 100755 src/bin/configure-fti-search.php
copy src/db/{20140304-fix_update_vectors_function.sql =>
20160502-fti-for-project-tags.sql} (83%)
create mode 100644 src/db/20160503-configurable-fti-config.sql
hooks/post-receive
--
FusionForge
_______________________________________________
Fusionforge-commits mailing list
[email protected]
http://lists.fusionforge.org/cgi-bin/mailman/listinfo/fusionforge-commits
