This is an automated email from the ASF dual-hosted git repository. cmeier pushed a commit to branch clojure-bert-qa-example in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit 459c8d92ef033a33ac44498445ef2262d9ef6fd4 Author: gigasquid <[email protected]> AuthorDate: Fri Apr 12 19:52:16 2019 -0400 try another question --- .../examples/bert-qa/src/bert_qa/core.clj | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj b/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj index 02f2d34..e44ad40 100644 --- a/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj +++ b/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj @@ -16,9 +16,11 @@ ;; the vocabulary used in the model (def model-vocab "model/vocab.json") ;; the input question -(def input-q "When did BBC Japan start broadcasting?") +#_(def input-q "When did BBC Japan start broadcasting?") +(def input-q "When did Herve serve as a Byzantine general?") ;;; the input answer -(def input-a (str "BBC Japan was a general entertainment Channel.\n" +(def input-a "One of the first Norman mercenaries to serve as a Byzantine general was Hervé in the 1050s. By then however, there were already Norman mercenaries serving as far away as Trebizond and Georgia. They were based at Malatya and Edessa, under the Byzantine duke of Antioch, Isaac Komnenos. In the 1060s, Robert Crispin led the Normans of Edessa against the Turks. Roussel de Bailleul even tried to carve out an independent state in Asia Minor with support from the local population, [...] +#_(def input-a (str "BBC Japan was a general entertainment Channel.\n" " Which operated between December 2004 and April 2006.\n" "It ceased operations after its Japanese distributor folded.")) ;; the maximum length of the sequence @@ -51,7 +53,7 @@ :token2idx (get vocab "token_to_idx")})) (defn tokens->idxs [token2idx tokens] - (mapv #(get token2idx % "[UNK]") tokens)) + (mapv #(get token2idx % (get token2idx "[UNK]")) tokens)) (defn idxs->tokens [idx2token idxs] (mapv #(get idx2token %) idxs)) @@ -70,10 +72,15 @@ end-idx (-> (ndarray/argmax end-prob 1) (ndarray/->vec) (first))] - (subvec tokens (dec start-idx) (inc end-idx)))) + (println "start-idx" start-idx "end-idx" end-idx) + (if (> end-idx start-idx) + (subvec tokens (dec start-idx) (inc end-idx)) + (subvec tokens (dec end-idx) (inc end-idx)) ) +)) (defn infer [ctx] - (let [;;; pre-processing tokenize sentence + (let [ctx (context/default-context) + ;;; pre-processing tokenize sentence token-q (tokenizer (string/lower-case input-q)) token-a (tokenizer (string/lower-case input-a)) valid-length (+ (count token-q) (count token-a)) @@ -126,5 +133,7 @@ (comment - (infer) + (infer :cpu) + + (get (:token2idx (get-vocab)) "[UNK]") )
