This is an automated email from the ASF dual-hosted git repository. LinkinStars pushed a commit to branch dev in repository https://gitbox.apache.org/repos/asf/answer-plugins.git
commit b8244a32ec9d00ab9e870a1a248502ae5b9bb1a5 Author: hgaol <[email protected]> AuthorDate: Mon May 11 22:20:46 2026 +0800 feat(pgvector): add PostgreSQL pgvector vector search plugin with configuration and sync functionality --- vector-search-pgvector/README.md | 49 ++++ vector-search-pgvector/go.mod | 11 + vector-search-pgvector/i18n/en_US.yaml | 61 +++++ vector-search-pgvector/i18n/translation.go | 45 ++++ vector-search-pgvector/i18n/zh_CN.yaml | 61 +++++ vector-search-pgvector/info.yaml | 22 ++ vector-search-pgvector/pgvector.go | 398 +++++++++++++++++++++++++++++ vector-search-pgvector/sync.go | 105 ++++++++ 8 files changed, 752 insertions(+) diff --git a/vector-search-pgvector/README.md b/vector-search-pgvector/README.md new file mode 100644 index 0000000..ce07404 --- /dev/null +++ b/vector-search-pgvector/README.md @@ -0,0 +1,49 @@ +# Apache Answer Plugin: pgvector Vector Search + +This plugin enables semantic/vector search in [Apache Answer](https://github.com/apache/answer) using [PostgreSQL](https://www.postgresql.org/) with the [pgvector](https://github.com/pgvector/pgvector) extension. + +## Prerequisites + +- PostgreSQL 12+ with the [pgvector](https://github.com/pgvector/pgvector) extension installed +- An OpenAI-compatible embedding API (e.g., OpenAI, Azure OpenAI, or any compatible provider) + +## Installation + +Build Apache Answer with this plugin: + +```bash +./answer build --with github.com/apache/answer-plugins/vector-search-pgvector +``` + +## Configuration + +After enabling the plugin in the Admin UI (**Admin > Plugins > Vector Search**), configure the following fields: + +| Field | Description | Example | +|---|---|---| +| **DSN** | PostgreSQL connection string | `postgres://user:pass@localhost:5432/answer?sslmode=disable` | +| **Embedding API Host** | OpenAI-compatible API base URL | `https://api.openai.com` | +| **Embedding API Key** | API key for the embedding service | `sk-...` | +| **Embedding Model** | Model name for generating embeddings | `text-embedding-3-small` | +| **Embedding Level** | `question` embeds question + all answers + comments together; `answer` embeds each answer separately | `question` | +| **Similarity Threshold** | Minimum cosine similarity score (0-1). Default `0` means no filtering | `0.5` | + +## How It Works + +- **Embedding dimensions** are auto-detected from the configured model. No manual dimension configuration is needed. +- On first configuration, the plugin creates a table `answer_vector_embeddings` with a `vector` column matching the detected dimensions. +- If the embedding model changes and produces different dimensions, the table is automatically dropped and recreated. +- Uses cosine similarity (`<=>` operator) for vector search. +- A full sync of all questions/answers is triggered when the plugin starts. + +## Database Setup + +Ensure the pgvector extension is installed in your PostgreSQL database: + +```sql +CREATE EXTENSION IF NOT EXISTS vector; +``` + +## License + +[Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) diff --git a/vector-search-pgvector/go.mod b/vector-search-pgvector/go.mod new file mode 100644 index 0000000..e0fbfe3 --- /dev/null +++ b/vector-search-pgvector/go.mod @@ -0,0 +1,11 @@ +module github.com/apache/answer-plugins/vector-search-pgvector + +go 1.23.0 + +require ( + github.com/apache/answer v1.7.0 + github.com/apache/answer-plugins/util v1.0.3-0.20250107030257-cf94ebc70954 + github.com/lib/pq v1.10.9 + github.com/pgvector/pgvector-go v0.2.2 + github.com/segmentfault/pacman v1.0.5-0.20230822083413-c0075a2d401f +) diff --git a/vector-search-pgvector/i18n/en_US.yaml b/vector-search-pgvector/i18n/en_US.yaml new file mode 100644 index 0000000..6059eb3 --- /dev/null +++ b/vector-search-pgvector/i18n/en_US.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +plugin: + pgvector_search: + backend: + info: + name: + other: PostgreSQL pgvector Search + description: + other: Use PostgreSQL with pgvector extension for semantic/vector search + config: + dsn: + title: + other: PostgreSQL DSN + description: + other: "PostgreSQL connection string (e.g. postgres://user:pass@localhost:5432/dbname?sslmode=disable)" + api_host: + title: + other: Embedding API Host + description: + other: "OpenAI-compatible API base URL for embedding generation (e.g. https://api.openai.com)" + api_key: + title: + other: Embedding API Key + description: + other: API key for the embedding service + embedding_model: + title: + other: Embedding Model + description: + other: "Model name for embedding generation (e.g. text-embedding-3-small)" + embedding_level: + title: + other: Embedding Level + description: + other: "Choose 'question' to embed question + all answers + comments, or 'answer' to embed each answer separately" + options: + question: + other: Question + answer: + other: Answer + similarity_threshold: + title: + other: Similarity Threshold + description: + other: "Minimum cosine similarity score (0-1) for search results. Default 0 (no filtering)." diff --git a/vector-search-pgvector/i18n/translation.go b/vector-search-pgvector/i18n/translation.go new file mode 100644 index 0000000..254a19a --- /dev/null +++ b/vector-search-pgvector/i18n/translation.go @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package i18n + +const ( + InfoName = "plugin.pgvector_search.backend.info.name" + InfoDescription = "plugin.pgvector_search.backend.info.description" + + ConfigDSNTitle = "plugin.pgvector_search.backend.config.dsn.title" + ConfigDSNDescription = "plugin.pgvector_search.backend.config.dsn.description" + + ConfigAPIHostTitle = "plugin.pgvector_search.backend.config.api_host.title" + ConfigAPIHostDescription = "plugin.pgvector_search.backend.config.api_host.description" + + ConfigAPIKeyTitle = "plugin.pgvector_search.backend.config.api_key.title" + ConfigAPIKeyDescription = "plugin.pgvector_search.backend.config.api_key.description" + + ConfigEmbeddingModelTitle = "plugin.pgvector_search.backend.config.embedding_model.title" + ConfigEmbeddingModelDescription = "plugin.pgvector_search.backend.config.embedding_model.description" + + ConfigEmbeddingLevelTitle = "plugin.pgvector_search.backend.config.embedding_level.title" + ConfigEmbeddingLevelDescription = "plugin.pgvector_search.backend.config.embedding_level.description" + ConfigEmbeddingLevelOptionQuestion = "plugin.pgvector_search.backend.config.embedding_level.options.question" + ConfigEmbeddingLevelOptionAnswer = "plugin.pgvector_search.backend.config.embedding_level.options.answer" + + ConfigSimilarityThresholdTitle = "plugin.pgvector_search.backend.config.similarity_threshold.title" + ConfigSimilarityThresholdDescription = "plugin.pgvector_search.backend.config.similarity_threshold.description" +) diff --git a/vector-search-pgvector/i18n/zh_CN.yaml b/vector-search-pgvector/i18n/zh_CN.yaml new file mode 100644 index 0000000..66887d8 --- /dev/null +++ b/vector-search-pgvector/i18n/zh_CN.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +plugin: + pgvector_search: + backend: + info: + name: + other: PostgreSQL pgvector 搜索 + description: + other: 使用 PostgreSQL pgvector 扩展进行语义/向量搜索 + config: + dsn: + title: + other: PostgreSQL 连接字符串 + description: + other: "PostgreSQL 连接字符串(如 postgres://user:pass@localhost:5432/dbname?sslmode=disable)" + api_host: + title: + other: Embedding API 地址 + description: + other: "用于生成 Embedding 的 OpenAI 兼容 API 地址(如 https://api.openai.com)" + api_key: + title: + other: Embedding API 密钥 + description: + other: Embedding 服务的 API 密钥 + embedding_model: + title: + other: Embedding 模型 + description: + other: "Embedding 模型名称(如 text-embedding-3-small)" + embedding_level: + title: + other: Embedding 级别 + description: + other: "选择 'question' 将问题+所有回答+评论合并嵌入,或 'answer' 将每个回答单独嵌入" + options: + question: + other: 问题 + answer: + other: 回答 + similarity_threshold: + title: + other: 相似度阈值 + description: + other: "搜索结果的最低余弦相似度分数(0-1)。默认为 0(不过滤)。" diff --git a/vector-search-pgvector/info.yaml b/vector-search-pgvector/info.yaml new file mode 100644 index 0000000..b9b2f60 --- /dev/null +++ b/vector-search-pgvector/info.yaml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +slug_name: pgvector_search +type: vector_search +version: 0.1.0 +author: answerdev +link: https://github.com/apache/answer-plugins/tree/main/vector-search-pgvector diff --git a/vector-search-pgvector/pgvector.go b/vector-search-pgvector/pgvector.go new file mode 100644 index 0000000..dc50601 --- /dev/null +++ b/vector-search-pgvector/pgvector.go @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package pgvector_search + +import ( + "context" + "database/sql" + "embed" + "encoding/json" + "fmt" + "strconv" + "sync" + + "github.com/apache/answer-plugins/util" + "github.com/apache/answer-plugins/vector-search-pgvector/i18n" + "github.com/apache/answer/plugin" + _ "github.com/lib/pq" + pgv "github.com/pgvector/pgvector-go" + "github.com/segmentfault/pacman/log" +) + +//go:embed info.yaml +var Info embed.FS + +const tableName = "answer_vector_embeddings" + +// VectorSearchEngine implements plugin.VectorSearch using PostgreSQL + pgvector. +type VectorSearchEngine struct { + Config *VectorSearchConfig + db *sql.DB + syncer plugin.VectorSearchSyncer + syncing bool + lock sync.Mutex + embeddingDimensions int // auto-detected from embedding model +} + +// VectorSearchConfig holds all plugin configuration. +type VectorSearchConfig struct { + DSN string `json:"dsn"` + APIHost string `json:"api_host"` + APIKey string `json:"api_key"` + EmbeddingModel string `json:"embedding_model"` + EmbeddingLevel string `json:"embedding_level"` + SimilarityThreshold float64 `json:"similarity_threshold"` +} + +func init() { + plugin.Register(&VectorSearchEngine{ + Config: &VectorSearchConfig{}, + lock: sync.Mutex{}, + }) +} + +func (e *VectorSearchEngine) Info() plugin.Info { + info := &util.Info{} + info.GetInfo(Info) + + return plugin.Info{ + Name: plugin.MakeTranslator(i18n.InfoName), + SlugName: info.SlugName, + Description: plugin.MakeTranslator(i18n.InfoDescription), + Author: info.Author, + Version: info.Version, + Link: info.Link, + } +} + +// Description returns metadata about this vector search engine. +func (e *VectorSearchEngine) Description() plugin.VectorSearchDesc { + return plugin.VectorSearchDesc{ + Icon: "", + Link: "https://github.com/pgvector/pgvector", + } +} + +// RegisterSyncer stores the syncer and triggers a full sync. +func (e *VectorSearchEngine) RegisterSyncer(ctx context.Context, syncer plugin.VectorSearchSyncer) { + log.Debugf("pgvector: RegisterSyncer called, db=%v, embeddingLevel=%s", e.db != nil, e.Config.EmbeddingLevel) + e.syncer = syncer + if e.db != nil { + e.sync() + } +} + +// SearchSimilar performs a cosine similarity search using pgvector. +func (e *VectorSearchEngine) SearchSimilar(ctx context.Context, query string, topK int) ([]plugin.VectorSearchResult, error) { + if e.db == nil { + return nil, fmt.Errorf("pgvector: database not initialized") + } + if topK <= 0 { + topK = 10 + } + + log.Debugf("pgvector: SearchSimilar query=%q topK=%d model=%s", query, topK, e.Config.EmbeddingModel) + + embedding, err := plugin.GenerateEmbedding(ctx, e.Config.APIHost, e.Config.APIKey, e.Config.EmbeddingModel, query) + if err != nil { + return nil, fmt.Errorf("generate query embedding: %w", err) + } + + log.Debugf("pgvector: search embedding generated, dimensions=%d", len(embedding)) + + vec := pgv.NewVector(embedding) + + // Use cosine distance operator (<=>). Score = 1 - distance. + sqlQuery := fmt.Sprintf( + `SELECT object_id, object_type, metadata, 1 - (embedding <=> $1) AS score + FROM %s + ORDER BY embedding <=> $1 + LIMIT $2`, tableName) + + rows, err := e.db.QueryContext(ctx, sqlQuery, vec, topK) + if err != nil { + return nil, fmt.Errorf("pgvector search failed: %w", err) + } + defer rows.Close() + + results := make([]plugin.VectorSearchResult, 0, topK) + for rows.Next() { + var r plugin.VectorSearchResult + if err := rows.Scan(&r.ObjectID, &r.ObjectType, &r.Metadata, &r.Score); err != nil { + log.Warnf("pgvector: scan row: %v", err) + continue + } + if e.Config.SimilarityThreshold > 0 && r.Score < e.Config.SimilarityThreshold { + log.Debugf("pgvector: skipping result %s score=%.4f below threshold=%.4f", r.ObjectID, r.Score, e.Config.SimilarityThreshold) + continue + } + results = append(results, r) + } + log.Debugf("pgvector: SearchSimilar returning %d results", len(results)) + return results, rows.Err() +} + +// UpdateContent upserts a single document into the pgvector table. +func (e *VectorSearchEngine) UpdateContent(ctx context.Context, content *plugin.VectorSearchContent) error { + if e.db == nil { + return fmt.Errorf("pgvector: database not initialized") + } + + log.Debugf("pgvector: UpdateContent objectID=%s objectType=%s titleLen=%d contentLen=%d", + content.ObjectID, content.ObjectType, len(content.Title), len(content.Content)) + + embedding, err := plugin.GenerateEmbedding(ctx, e.Config.APIHost, e.Config.APIKey, e.Config.EmbeddingModel, content.Content) + if err != nil { + return fmt.Errorf("generate embedding for %s: %w", content.ObjectID, err) + } + + log.Debugf("pgvector: embedding generated for %s, dimensions=%d", content.ObjectID, len(embedding)) + + vec := pgv.NewVector(embedding) + + query := fmt.Sprintf( + `INSERT INTO %s (object_id, object_type, title, content, metadata, embedding) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (object_id) DO UPDATE SET + object_type = EXCLUDED.object_type, + title = EXCLUDED.title, + content = EXCLUDED.content, + metadata = EXCLUDED.metadata, + embedding = EXCLUDED.embedding`, tableName) + + _, err = e.db.ExecContext(ctx, query, + content.ObjectID, content.ObjectType, content.Title, content.Content, content.Metadata, vec) + if err != nil { + return fmt.Errorf("upsert document %s: %w", content.ObjectID, err) + } + log.Debugf("pgvector: upserted document %s successfully", content.ObjectID) + return nil +} + +// DeleteContent removes a document by object ID. +func (e *VectorSearchEngine) DeleteContent(ctx context.Context, objectID string) error { + if e.db == nil { + return fmt.Errorf("pgvector: database not initialized") + } + log.Debugf("pgvector: DeleteContent objectID=%s", objectID) + _, err := e.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM %s WHERE object_id = $1`, tableName), objectID) + if err != nil { + return fmt.Errorf("delete document %s: %w", objectID, err) + } + log.Debugf("pgvector: deleted document %s", objectID) + return nil +} + +// ConfigFields returns the plugin configuration form fields. +func (e *VectorSearchEngine) ConfigFields() []plugin.ConfigField { + return []plugin.ConfigField{ + { + Name: "dsn", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigDSNTitle), + Description: plugin.MakeTranslator(i18n.ConfigDSNDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: e.Config.DSN, + }, + { + Name: "api_host", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigAPIHostTitle), + Description: plugin.MakeTranslator(i18n.ConfigAPIHostDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: e.Config.APIHost, + }, + { + Name: "api_key", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigAPIKeyTitle), + Description: plugin.MakeTranslator(i18n.ConfigAPIKeyDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypePassword, + }, + Value: e.Config.APIKey, + }, + { + Name: "embedding_model", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigEmbeddingModelTitle), + Description: plugin.MakeTranslator(i18n.ConfigEmbeddingModelDescription), + Required: true, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: e.Config.EmbeddingModel, + }, + { + Name: "embedding_level", + Type: plugin.ConfigTypeSelect, + Title: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelTitle), + Description: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelDescription), + Required: true, + Options: []plugin.ConfigFieldOption{ + {Label: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelOptionQuestion), Value: "question"}, + {Label: plugin.MakeTranslator(i18n.ConfigEmbeddingLevelOptionAnswer), Value: "answer"}, + }, + Value: e.Config.EmbeddingLevel, + }, + { + Name: "similarity_threshold", + Type: plugin.ConfigTypeInput, + Title: plugin.MakeTranslator(i18n.ConfigSimilarityThresholdTitle), + Description: plugin.MakeTranslator(i18n.ConfigSimilarityThresholdDescription), + Required: false, + UIOptions: plugin.ConfigFieldUIOptions{ + InputType: plugin.InputTypeText, + }, + Value: fmt.Sprintf("%.2f", e.Config.SimilarityThreshold), + }, + } +} + +// ConfigReceiver applies configuration from the admin UI. +func (e *VectorSearchEngine) ConfigReceiver(config []byte) error { + log.Debugf("pgvector: ConfigReceiver called, raw config=%s", string(config)) + + // Pre-process: convert string numbers to actual numbers before unmarshalling, + // because the admin UI sends all form values as strings. + var raw map[string]interface{} + if err := json.Unmarshal(config, &raw); err != nil { + return fmt.Errorf("unmarshal config: %w", err) + } + if threshStr, ok := raw["similarity_threshold"].(string); ok { + if thresh, err := strconv.ParseFloat(threshStr, 64); err == nil { + raw["similarity_threshold"] = thresh + } + } + fixed, _ := json.Marshal(raw) + + conf := &VectorSearchConfig{} + if err := json.Unmarshal(fixed, conf); err != nil { + return fmt.Errorf("unmarshal config: %w", err) + } + + e.Config = conf + + log.Debugf("pgvector: config parsed: dsn=%s model=%s level=%s threshold=%.2f", + conf.DSN, conf.EmbeddingModel, conf.EmbeddingLevel, conf.SimilarityThreshold) + + if !plugin.StatusManager.IsEnabled("pgvector_search") { + log.Debugf("pgvector: plugin not active, skipping initialization") + return nil + } + + // Auto-detect embedding dimensions by generating a probe embedding. + log.Debugf("pgvector: detecting embedding dimensions via probe call") + probeEmbedding, err := plugin.GenerateEmbedding(context.Background(), conf.APIHost, conf.APIKey, conf.EmbeddingModel, "dimension probe") + if err != nil { + return fmt.Errorf("detect embedding dimensions: %w", err) + } + e.embeddingDimensions = len(probeEmbedding) + log.Infof("pgvector: auto-detected embedding dimensions=%d for model=%s", e.embeddingDimensions, conf.EmbeddingModel) + + log.Debugf("pgvector: connecting to PostgreSQL") + + db, err := sql.Open("postgres", conf.DSN) + if err != nil { + return fmt.Errorf("open postgres: %w", err) + } + if err := db.Ping(); err != nil { + db.Close() + return fmt.Errorf("ping postgres: %w", err) + } + + log.Debugf("pgvector: connected to PostgreSQL successfully") + + // Close previous connection if any. + if e.db != nil { + e.db.Close() + } + e.db = db + + if err := e.ensureTable(context.Background()); err != nil { + return fmt.Errorf("ensure table: %w", err) + } + + log.Debugf("pgvector: ConfigReceiver completed successfully") + return nil +} + +// ensureTable creates the pgvector extension and the embeddings table if they don't exist. +// If the table exists but the embedding dimensions differ, it drops and recreates the table. +func (e *VectorSearchEngine) ensureTable(ctx context.Context) error { + // Enable the pgvector extension. + if _, err := e.db.ExecContext(ctx, `CREATE EXTENSION IF NOT EXISTS vector`); err != nil { + return fmt.Errorf("create vector extension: %w", err) + } + + dims := e.embeddingDimensions + if dims <= 0 { + dims = 1536 + } + + log.Debugf("pgvector: ensureTable with dimensions=%d", dims) + + // Check if table exists and verify embedding dimensions match. + var existingDims int + err := e.db.QueryRowContext(ctx, fmt.Sprintf( + `SELECT atttypmod FROM pg_attribute + WHERE attrelid = '%s'::regclass AND attname = 'embedding'`, tableName)).Scan(&existingDims) + if err == nil && existingDims > 0 && existingDims != dims { + log.Warnf("pgvector: embedding dimensions changed from %d to %d, recreating table", existingDims, dims) + if _, err := e.db.ExecContext(ctx, fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tableName)); err != nil { + return fmt.Errorf("drop table for dimension change: %w", err) + } + } else if err != nil { + log.Debugf("pgvector: table does not exist yet or dimension check skipped: %v", err) + } else { + log.Debugf("pgvector: existing table dimensions=%d, matches configured=%d", existingDims, dims) + } + + createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + object_id TEXT PRIMARY KEY, + object_type TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', + metadata TEXT NOT NULL DEFAULT '', + embedding vector(%d) NOT NULL + )`, tableName, dims) + + if _, err := e.db.ExecContext(ctx, createSQL); err != nil { + return fmt.Errorf("create table: %w", err) + } + + // Create an IVFFlat index for faster cosine similarity search if it doesn't exist. + indexSQL := fmt.Sprintf( + `CREATE INDEX IF NOT EXISTS idx_%s_embedding ON %s USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)`, + tableName, tableName) + if _, err := e.db.ExecContext(ctx, indexSQL); err != nil { + // IVFFlat requires data in the table to build the index; log and continue if empty. + log.Warnf("pgvector: create ivfflat index (may need data first): %v", err) + } + + return nil +} diff --git a/vector-search-pgvector/sync.go b/vector-search-pgvector/sync.go new file mode 100644 index 0000000..e4a3063 --- /dev/null +++ b/vector-search-pgvector/sync.go @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package pgvector_search + +import ( + "context" + + "github.com/apache/answer/plugin" + "github.com/segmentfault/pacman/log" +) + +// sync triggers a full bulk sync of all Q&A content into the pgvector table. +// It runs in a background goroutine and is guarded by the syncing flag. +func (e *VectorSearchEngine) sync() { + if e.syncer == nil { + log.Warn("pgvector: syncer not registered, skip sync") + return + } + if e.syncing { + log.Warn("pgvector: sync already running, skip") + return + } + + go func() { + e.lock.Lock() + defer e.lock.Unlock() + if e.syncing { + return + } + e.syncing = true + defer func() { e.syncing = false }() + + page, pageSize := 1, 100 + + if e.Config.EmbeddingLevel == "" || e.Config.EmbeddingLevel == "question" { + log.Info("pgvector: starting question sync...") + page = 1 + for { + log.Infof("pgvector: sync questions page %d", page) + questions, err := e.syncer.GetQuestionsPage(context.TODO(), page, pageSize) + if err != nil { + log.Errorf("pgvector: get questions page failed: %v", err) + break + } + if len(questions) == 0 { + break + } + e.bulkIndex(context.TODO(), questions) + page++ + } + } + + if e.Config.EmbeddingLevel == "answer" { + log.Info("pgvector: starting answer sync...") + page = 1 + for { + log.Infof("pgvector: sync answers page %d", page) + answers, err := e.syncer.GetAnswersPage(context.TODO(), page, pageSize) + if err != nil { + log.Errorf("pgvector: get answers page failed: %v", err) + break + } + if len(answers) == 0 { + break + } + e.bulkIndex(context.TODO(), answers) + page++ + } + } + + log.Info("pgvector: sync complete") + }() +} + +// bulkIndex indexes a batch of documents, computing embeddings for each. +func (e *VectorSearchEngine) bulkIndex(ctx context.Context, contents []*plugin.VectorSearchContent) { + log.Debugf("pgvector: bulkIndex batch size=%d", len(contents)) + success, failed := 0, 0 + for _, c := range contents { + if err := e.UpdateContent(ctx, c); err != nil { + log.Warnf("pgvector: index %s failed: %v", c.ObjectID, err) + failed++ + } else { + success++ + } + } + log.Debugf("pgvector: bulkIndex batch done, success=%d failed=%d", success, failed) +}
