This is an automated email from the git hooks/post-receive script.
guix_mirror_bot pushed a commit to branch master
in repository guix.
The following commit(s) were added to refs/heads/master by this push:
new 98df55c929 gnu: Add python-tokenizer.
98df55c929 is described below
commit 98df55c929cd77d5d62e873ab25d178a5b3c9790
Author: Spencer King <[email protected]>
AuthorDate: Fri Jul 17 19:10:38 2026 +0000
gnu: Add python-tokenizer.
* gnu/packages/language.scm (python-tokenizer): New variable.
Merges: https://codeberg.org/guix/guix/pulls/10009
Signed-off-by: Nguyễn Gia Phong <[email protected]>
---
gnu/packages/language.scm | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/gnu/packages/language.scm b/gnu/packages/language.scm
index cd603c57a7..cf9cfb3a27 100644
--- a/gnu/packages/language.scm
+++ b/gnu/packages/language.scm
@@ -1350,6 +1350,28 @@ instance in spelling correction, predictive typing, to
help disabled people
write text fast, and for various text generation, statistics, and modeling
tasks.")
(license license:expat)))
+(define-public python-tokenizer
+ (package
+ (name "python-tokenizer")
+ (version "3.6.4")
+ (source
+ (origin
+ (method git-fetch)
+ (uri (git-reference
+ (url "https://github.com/mideind/Tokenizer";)
+ (commit (string-append "v" version))))
+ (file-name (git-file-name name version))
+ (sha256
+ (base32 "0acfy1aqf1581vy4d5y6wkjfsx52c6azxx2hy85x9rqv6fpij0m2"))))
+ (build-system pyproject-build-system)
+ (native-inputs (list python-pytest
+ python-setuptools))
+ (home-page "https://github.com/mideind/Tokenizer";)
+ (synopsis "Tokenizer for Icelandic text with sentence segmentation")
+ (description "This package provides a fast, compact, pure-Python tokenizer
+for Icelandic text with sentence segmentation.")
+ (license license:expat)))
+
(define-public python-sacremoses
(package
(name "python-sacremoses")
