bitflicker64 commented on code in PR #405: URL: https://github.com/apache/hugegraph-ai/pull/405#discussion_r4505778518
########## hugegraph-llm-sdk/eval/src/hugegraph_llm_eval/integrations/judges/codex_app_server.py: ########## @@ -0,0 +1,446 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Judge backed by ``codex app-server``, so evaluation can run on a ChatGPT/Codex subscription.""" + +import os +import queue +import shutil +import tempfile +import threading +import time +import weakref +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from types import TracebackType +from typing import Literal, Self + +from pydantic import JsonValue, ValidationError + +from hugegraph_llm_eval.contracts.domain.judge import ProviderUsage +from hugegraph_llm_eval.contracts.errors import FailureCategory, FailureDetail +from hugegraph_llm_eval.evaluation.judges.base import JudgeDescriptor, JudgeRequest, OutputT +from hugegraph_llm_eval.integrations.judges.attempts import ( + AttemptFailureError, + AttemptLoopJudge, + AttemptOutcome, + failure, +) +from hugegraph_llm_eval.integrations.judges.cli_common import ( + cli_version, + split_messages, + strict_json_schema, + token_count, +) +from hugegraph_llm_eval.integrations.judges.codex_connection import ( + AppServerConnection, + AppServerProcess, + ConnectionClosedError, + RpcError, + spawn_app_server, +) +from hugegraph_llm_eval.utils.clock import utc_now + +PROVIDER = "codex-app-server" +_STARTUP_TIMEOUT_SECONDS = 60.0 +_TRANSIENT_KINDS = frozenset({"serverOverloaded", "internalServerError", "flexUnavailable"}) +_TRANSIENT_PREFIXES = ("httpConnectionFailed", "responseStream") +_DEFAULT_INSTRUCTIONS = "You are an evaluation judge. Do not use tools. Answer only with the requested JSON." +_TOOL_FEATURES = ("plugins", "apps", "computer_use", "browser_use", "shell_tool", "unified_exec", "multi_agent") +_DEFAULT_CONFIG: Mapping[str, JsonValue] = { + "include_environment_context": False, + "include_permissions_instructions": False, + "include_apps_instructions": False, + "project_doc_max_bytes": 0, + "web_search": "disabled", +} +_SUBSCRIPTION_ENV_NAMES = frozenset( + { + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "OPENAI_ORGANIZATION", + "OPENAI_PROJECT", + "CODEX_API_KEY", + "CODEX_ACCESS_TOKEN", + "OPENAI_FEDERATION_RULE_ID", + "OPENAI_IDENTITY_TOKEN_FILE", + "OPENAI_WORKLOAD_IDENTITY_CONTEXT", + } +) + + +class _ConnectionHolder: + """Owns the shared connection; its ``close`` also serves as the finalizer.""" + + def __init__(self) -> None: + self.connection: AppServerConnection | None = None + + def close(self) -> None: + connection, self.connection = self.connection, None + if connection is not None: + connection.close() + + +class CodexAppServerJudge(AttemptLoopJudge): + """Runs Judge calls as ephemeral threads on one shared ``codex app-server`` process.""" + + def __init__( + self, + *, + model: str, + effort: str | None = None, + executable: str = "codex", + config: Mapping[str, JsonValue] | None = None, + auth_mode: Literal["inherit", "subscription"] = "inherit", + disable_user_mcp_servers: bool = True, + call_timeout_seconds: float = 600.0, + version: str | None = None, + clock: Callable[[], datetime] = utc_now, + spawn: Callable[..., AppServerProcess] = spawn_app_server, + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + if call_timeout_seconds <= 0: + msg = "call_timeout_seconds must be positive" + raise ValueError(msg) + if auth_mode not in ("inherit", "subscription"): + msg = "auth_mode must be 'inherit' or 'subscription'" + raise ValueError(msg) + provided = dict(config or {}) + if "mcp_servers" in provided: + raise ValueError("config.mcp_servers cannot be set for an evaluation judge") + features = provided.pop("features", {}) + if provided: + keys = ", ".join(sorted(provided)) + raise ValueError(f"unsupported raw Codex config keys: {keys}; use typed options") + if not isinstance(features, Mapping) or not all(isinstance(name, str) for name in features): + raise ValueError("config.features must be a string-keyed mapping") + unknown_features = sorted(set(features) - set(_TOOL_FEATURES)) + if unknown_features: + raise ValueError(f"unsupported Codex feature flags: {', '.join(unknown_features)}") + unsafe_features = [name for name, enabled in features.items() if enabled is not False] + if unsafe_features: + raise ValueError(f"Codex judge cannot enable tool capabilities: {', '.join(sorted(unsafe_features))}") + if not disable_user_mcp_servers: + msg = "disable_user_mcp_servers cannot be false for an evaluation judge" + raise ValueError(msg) + merged: dict[str, JsonValue] = { + **_DEFAULT_CONFIG, + "features": {**dict.fromkeys(_TOOL_FEATURES, False), **features}, + } + configuration: JsonValue = { + "cli_version": version if version is not None else cli_version(executable), + "effort": effort, + "auth_mode": auth_mode, + "route": "openai-chatgpt-account" if auth_mode == "subscription" else "inherited", + "config": merged, + "disable_user_mcp_servers": True, + "sandbox": "read-only", + "sampling": "provider-controlled", + } + super().__init__( + descriptor=JudgeDescriptor.create(provider=PROVIDER, model=model, configuration=configuration), + clock=clock, + monotonic=monotonic, + sleep=sleep, + ) + self._executable = executable + self._effort = effort + self._config = merged + self._auth_mode = auth_mode + self._call_timeout = call_timeout_seconds + self._spawn = spawn + self._mcp_overrides: dict[str, JsonValue] = {} + self._lock = threading.Lock() + self._holder = _ConnectionHolder() + self._workdir = tempfile.mkdtemp(prefix="hugegraph-llm-eval-codex-") + self._isolated_codex_home = Path(self._workdir) / "codex-home" + self._isolated_codex_home.mkdir(mode=0o700) + weakref.finalize(self, shutil.rmtree, self._workdir, ignore_errors=True) + weakref.finalize(self, self._holder.close) + + def close(self) -> None: + with self._lock: + self._holder.close() + + def __enter__(self) -> Self: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.close() + + def _connection(self, timeout: float) -> AppServerConnection: + with self._lock: + connection = self._holder.connection + if connection is None or not connection.alive: + self._holder.close() + connection = self._start(min(timeout, _STARTUP_TIMEOUT_SECONDS)) + self._holder.connection = connection + return connection + + def _start(self, timeout: float) -> AppServerConnection: + process = self._spawn([self._executable, "app-server"], cwd=self._workdir, env=self._environment()) + connection = AppServerConnection(process) + try: + connection.request("initialize", {"clientInfo": {"name": "hugegraph-llm-eval", "version": "1"}}, timeout) + connection.notify("initialized") + if self._auth_mode == "subscription": + self._require_chatgpt_account(connection, timeout) + self._mcp_overrides = self._verified_isolation_overrides(connection, timeout) + except BaseException: + connection.close() + raise + return connection + + def _require_chatgpt_account(self, connection: AppServerConnection, timeout: float) -> None: + account = connection.request("account/read", {"refreshToken": False}, timeout).get("account") + if not isinstance(account, dict): + raise AttemptFailureError( + failure(FailureCategory.PROVIDER_UNAVAILABLE, "provider_not_authenticated", False) + ) + if account.get("type") != "chatgpt": + raise AttemptFailureError( + failure(FailureCategory.PROVIDER_UNAVAILABLE, "provider_account_not_chatgpt", False) + ) + + def _environment(self) -> dict[str, str]: + """Use a private Codex home so user hooks and config cannot enter an eval run.""" + env = dict(os.environ) + source_home = Path(env.get("CODEX_HOME", Path.home() / ".codex")).expanduser() + auth_file = source_home / "auth.json" + if auth_file.is_file(): + target_auth = self._isolated_codex_home / "auth.json" + shutil.copyfile(auth_file, target_auth) Review Comment: Important: Codex refresh tokens are single-use, so a refresh inside this private `CODEX_HOME` leaves `~/.codex/auth.json` holding a used token, the new one is deleted with the temp dir, and a restarted app-server recopies the stale file. Keep tokens in the user's `CODEX_HOME` and isolate through config overrides only, or copy a refreshed `auth.json` back before cleanup. ########## hugegraph-llm-sdk/eval/src/hugegraph_llm_eval/utils/text_normalize.py: ########## @@ -0,0 +1,158 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Text normalization utilities for text evaluation. + +Implements the MRQA official evaluation standard from HippoRAG 2 (eval_utils.py) +with Porter stemming and bilingual (EN/ZH) support. + +For Chinese text the normalization additionally: +- converts full-width ASCII characters (letters, digits, punctuation) to half-width +- performs simplified/traditional Chinese conversion when ``opencc`` is available +- removes Chinese punctuation and collapses whitespace +""" + +import re +import string +from typing import cast + +# Chinese punctuation set +_CHINESE_PUNCTUATION = set("，。！？、；：''（）【】《》〈〉…—～·「」『』〔〕") +# English stop words removed during normalization (MRQA standard) +_ARTICLES_PATTERN = re.compile(r"\b(a|an|the)\b") + +# Full-width ASCII block: U+FF01..U+FF5E map to U+0021..U+007E. +_FULLWIDTH_SPACE = " " # full-width space + + +def _to_halfwidth(text: str) -> str: + """Convert full-width ASCII characters to their half-width forms. + + Covers full-width letters, digits, punctuation and the full-width space. + This unifies mixed full/half-width text (e.g. Chinese manuals often contain + full-width numbers and letters) before comparison. + """ + # Full-width ASCII block U+FF01..U+FF5E maps to U+0021..U+007E. + table = {0xFF01 + i: 0x0021 + i for i in range(94)} + table[ord(_FULLWIDTH_SPACE)] = ord(" ") + return text.translate(table) + + +def _simplify_chinese(text: str) -> str: + """Convert traditional Chinese characters to simplified forms if possible. + + Uses ``opencc-python-reimplemented`` / ``opencc`` when installed. If the + library is not available the text is returned unchanged so evaluation + keeps working without extra dependencies. + """ + try: + # opencc-python-reimplemented exposes OpenCC in the same way + from opencc import OpenCC # type: ignore[import-not-found] # ty: ignore[import-not-found] + + converter = OpenCC("t2s") + return cast(str, converter.convert(text)) + except Exception: + return text + + +def normalize_answer(answer: str, language: str = "en") -> str: + """Normalize an answer string for comparison. + + Steps (EN): lowercase → remove punctuation → remove articles + (a/an/the) → collapse whitespace. + Steps (ZH): full-width to half-width → traditional to simplified Chinese + → lowercase → remove punctuation (incl. Chinese) → collapse whitespace. + + Reference: HippoRAG 2 / MRQA official eval_utils.normalize_answer + (standard SQuAD normalization: lowercase, remove punctuation, remove + a/an/the, collapse whitespace). Note: ``and`` is a conjunction, not an + article, and is intentionally NOT removed. + + Args: + answer: Raw answer text. + language: 'en' for English, 'zh' for Chinese. + + Returns: + Normalized string. + """ + if not answer: + return "" + + def _preprocess(text: str) -> str: + # Language-specific preprocessing before shared normalization. + if language == "zh": + text = _to_halfwidth(text) + text = _simplify_chinese(text) + return text + + def _lower(text: str) -> str: + return text.lower() + + def _remove_punc(text: str) -> str: + exclude = set(string.punctuation) | _CHINESE_PUNCTUATION + return "".join(ch for ch in text if ch not in exclude) + + def _remove_articles(text: str) -> str: + if language == "en": + return _ARTICLES_PATTERN.sub(" ", text) + return text + + def _white_space_fix(text: str) -> str: + return " ".join(text.split()) + + return _white_space_fix(_remove_articles(_remove_punc(_lower(_preprocess(answer))))) + + +def tokenize(text: str, language: str = "en", stem: bool = False) -> list[str]: + """Tokenize text into words, optionally with stemming. + + For English: split on whitespace after normalization. + For Chinese: use jieba segmentation. + + Args: + text: Raw text to tokenize. + language: 'en' or 'zh'. + stem: If True, apply Porter stemmer to English tokens (HippoRAG 2 standard). + + Returns: + List of tokens. + """ + if language == "zh": + import jieba # type: ignore[import-untyped] + + return list(jieba.cut(normalize_answer(text, language))) Review Comment: Important: `jieba.cut` keeps the spaces left by `normalize_answer` as tokens, so zh ROUGE-L matches whitespace: `_chinese_rouge("HugeGraph 支持 Gremlin", "Neo4j 支持 Cypher")` scores F1 0.6, against 0.3333 without the spaces. Drop whitespace-only tokens before returning. ########## hugegraph-llm-sdk/eval/src/hugegraph_llm_eval/integrations/retrieval/evaluators/coverage.py: ########## @@ -0,0 +1,332 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""LLM-judged context entity recall and evidence attribution Evaluators.""" + +from collections.abc import Callable +from datetime import datetime +from typing import Annotated + +from pydantic import Field, model_validator + +from hugegraph_llm_eval.contracts.base import StrictModel +from hugegraph_llm_eval.contracts.domain.artifacts import ArtifactRef +from hugegraph_llm_eval.contracts.domain.evaluation import ( + EvaluationContext, + EvaluationInput, + EvaluationRecordDraft, + EvidenceLink, + Finding, + FindingSeverity, +) +from hugegraph_llm_eval.contracts.errors import FailureCategory +from hugegraph_llm_eval.contracts.identity import ComponentRef +from hugegraph_llm_eval.evaluation.judges.base import Judge, JudgeCallError +from hugegraph_llm_eval.evaluation.judges.batching import BatchingOptions, BatchInputError, payload_batches +from hugegraph_llm_eval.evaluation.judges.content import JudgeContent +from hugegraph_llm_eval.evaluation.judges.session import JudgeSession +from hugegraph_llm_eval.integrations.retrieval.evaluators.judged_common import ( + JudgedRetrievalEvaluator, + indexed, + ratio, + require_unique_indices, + subject, +) +from hugegraph_llm_eval.integrations.retrieval.materializers import RetrievalArtifactMaterializer +from hugegraph_llm_eval.utils.text_preview import text_preview + +_ENTITY_EXTRACTION_INSTRUCTION = ( + "Extract every distinct entity from reference_answer, without deciding whether it is covered. " + 'Return only {"entities": ["entity name"]}; preserve named entities, include unsupported ones, ' + "and do not invent entities. The caller will freeze this inventory before evaluating retrieval." +) +_ENTITY_COVERAGE_INSTRUCTION = ( + "The supplied entities are the fixed reference inventory. Return has_reference_entities and verdicts. " + "For EVERY supplied entity_index return exactly one verdict with the same entity text, covered and reason. " + "Do not add, remove, rename or renumber entities, including those not covered by any retrieved context." +) + + +class ReferenceEntitiesOutput(StrictModel): + entities: tuple[Annotated[str, Field(min_length=1, max_length=4_000)], ...] + + @model_validator(mode="after") + def _unique_entities(self) -> "ReferenceEntitiesOutput": + if len(self.entities) != len(set(self.entities)): + raise ValueError("reference entity inventory must be unique") + return self + + +class ContextEntityVerdict(StrictModel): + """LLM verdict for one answer entity against all retrieved contexts.""" + + entity_index: int = Field(ge=0) + entity: str = Field(min_length=1, max_length=4_000) + covered: bool + reason: str = Field(min_length=1, max_length=4_000) + + +class ContextEntityRecallOutput(StrictModel): + """Semantic coverage judgements against a fixed reference entity inventory.""" + + has_reference_entities: bool + verdicts: tuple[ContextEntityVerdict, ...] + + @model_validator(mode="after") + def _complete_entity_decision(self) -> "ContextEntityRecallOutput": + require_unique_indices(tuple(item.entity_index for item in self.verdicts)) + if self.has_reference_entities != bool(self.verdicts): + raise ValueError("entity verdicts must agree with has_reference_entities") + return self + + +class EvidenceVerdict(StrictModel): + evidence_index: int = Field(ge=0) + attributed: bool + reason: str = Field(min_length=1, max_length=4_000) + + +class EvidenceRecallOutput(StrictModel): + verdicts: tuple[EvidenceVerdict, ...] + + @model_validator(mode="after") + def _unique_indices(self) -> "EvidenceRecallOutput": + require_unique_indices(tuple(item.evidence_index for item in self.verdicts)) + return self + + +class ContextEntityRecallEvaluator(JudgedRetrievalEvaluator): + def __init__( + self, + materializer: RetrievalArtifactMaterializer, + *, + judge: Judge, + prompt: JudgeContent, + rubric: JudgeContent, + clock: Callable[[], datetime], + batching: BatchingOptions | None = None, + ) -> None: + super().__init__( + materializer, + invocation_key="context_entity_recall_llm", + judge=judge, + prompt=prompt, + rubric=rubric, + clock=clock, + batching=batching, + implementation={ + "entity_protocol": "fixed-inventory-v1", + "inventory_prompt": _ENTITY_EXTRACTION_INSTRUCTION, + "coverage_prompt": _ENTITY_COVERAGE_INSTRUCTION, + }, + ) + + def evaluate(self, evaluation_input: EvaluationInput, context: EvaluationContext) -> EvaluationRecordDraft: + value = self.materialize(evaluation_input) + if isinstance(value, EvaluationRecordDraft): + return value + session = self.session(context) + supplied = value.reference.entities is not None + entities = value.reference.entities + try: + if entities is None: + entities = self._reference_entities(value.reference.answer, session) + if not entities: + return self.not_applicable_record("reference entity inventory is empty; recall is undefined", session) + verdicts: list[ContextEntityVerdict] = [] + batches = payload_batches( + tuple(enumerate(entities)), + lambda items: { + "question": value.query.question, + "entities": [{"entity_index": index, "text": text} for index, text in items], + "contexts": indexed(value.candidate.contexts, "context_index"), + }, + self._batching, + ) + for ordinal, (items, payload) in enumerate(batches): + key = "judge_entity_coverage" if ordinal == 0 else f"judge_entity_coverage_{ordinal}" + output = session.call( + key, + self.request( + ContextEntityRecallOutput, + payload, + prompt=self._entity_prompt("coverage", _ENTITY_COVERAGE_INSTRUCTION), + ), + ) + expected = dict(items) + if {item.entity_index for item in output.verdicts} != set(expected) or any( + expected.get(item.entity_index) != item.entity for item in output.verdicts + ): + return self.failure( + "incomplete_context_entity_verdicts", + "entity coverage must preserve every supplied entity and its global index", + FailureCategory.INVALID_PROVIDER_OUTPUT, + calls=session.calls, + ) + verdicts.extend(output.verdicts) + except JudgeCallError as exc: + return self.judge_failure(exc, session) + except BatchInputError as exc: + return self.failure( + "evaluation_payload_too_large", + str(exc), + FailureCategory.RESOURCE_EXHAUSTED, + calls=session.calls, + ) + ordered = sorted(verdicts, key=lambda item: item.entity_index) + covered = sum(item.covered for item in ordered) + artifact = subject(evaluation_input, "retrieval_reference") + source = "supplied" if supplied else "model-extracted" + return self.completed( + ratio("context_entity_recall_llm", round(covered / len(entities), 4)), + _entity_findings(ordered, artifact, supplied=supplied), + f"retrieved contexts cover {covered} of {len(entities)} entities in the fixed {source} inventory", + session, + ) + + def _reference_entities(self, answer: str, session: JudgeSession) -> tuple[str, ...]: + if not answer.strip(): + return () + batches = payload_batches((answer,), lambda items: {"reference_answer": items[0]}, self._batching) + _, payload = next(batches) + output = session.call( + "extract_reference_entities", + self.request( + ReferenceEntitiesOutput, + payload, + prompt=self._entity_prompt("inventory", _ENTITY_EXTRACTION_INSTRUCTION), + ), + ) + return output.entities + + def _entity_prompt(self, task: str, instruction: str) -> JudgeContent: + return JudgeContent( + component=ComponentRef(namespace="retrieval", name=f"entity_{task}", revision="v1"), + text=f"{self._prompt.text}\n\n{instruction}", + ) + + +def _entity_findings( + verdicts: list[ContextEntityVerdict], + artifact: ArtifactRef, + *, + supplied: bool, +) -> tuple[Finding, ...]: + """Render entity-coverage verdicts as item-level findings.""" + return tuple( + Finding( + code="context_entity_recall.covered" if verdict.covered else "context_entity_recall.missing", + severity=FindingSeverity.INFO if verdict.covered else FindingSeverity.ERROR, + message=text_preview(f"{verdict.entity}: {verdict.reason}", 4_000), + evidence=( + EvidenceLink( + artifact=artifact, + locator=( + f"entities[{verdict.entity_index}]" if supplied else f"answer.entity[{verdict.entity_index}]" + ), + description=text_preview(verdict.entity, 2_000), + ), + ), + ) + for verdict in verdicts + ) + + +class EvidenceRecallEvaluator(JudgedRetrievalEvaluator): + def __init__( + self, + materializer: RetrievalArtifactMaterializer, + *, + judge: Judge, + prompt: JudgeContent, + rubric: JudgeContent, + clock: Callable[[], datetime], + batching: BatchingOptions | None = None, + ) -> None: + super().__init__( + materializer, + invocation_key="evidence_recall_llm", + judge=judge, + prompt=prompt, + rubric=rubric, + clock=clock, + batching=batching, + ) + + def evaluate(self, evaluation_input: EvaluationInput, context: EvaluationContext) -> EvaluationRecordDraft: + value = self.materialize(evaluation_input) + if isinstance(value, EvaluationRecordDraft): + return value + session = self.session(context) + evidence = value.reference.evidence + artifact = subject(evaluation_input, "retrieval_reference") + if not evidence: + return self.completed(ratio("evidence_recall_llm", 1), (), "no reference evidence", session) Review Comment: Important: a sample without reference evidence gets a perfect `evidence_recall_llm` of 1.0 with no judge call, whatever was retrieved, and `retrieval_input` defaults evidence to `()`. Return `not_applicable_record` here, as context entity recall does for an empty inventory. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
