GNU bug report logs - #62443
LLaMA.cpp

Previous Next

Full log

Message #8 received at 62443 <at> debbugs.gnu.org (full text, mbox):

From: Nicolas Graves <ngraves <at> ngraves.fr>
To: 62443 <at> debbugs.gnu.org
Cc: ngraves <at> ngraves.fr
Subject: [PATCH 1/3] gnu: Add sentencepiece.
Date: Sat, 25 Mar 2023 16:32:18 +0100

* gnu/packages/machine-learning.scm (sentencepiece): New variable.
---
 gnu/packages/machine-learning.scm | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/gnu/packages/machine-learning.scm b/gnu/packages/machine-learning.scm
index 37d4ef78ad..f6996af77b 100644
--- a/gnu/packages/machine-learning.scm
+++ b/gnu/packages/machine-learning.scm
@@ -583,6 +583,33 @@ (define openfst-for-vosk
        '("--enable-shared" "--enable-far" "--enable-ngram-fsts"
          "--enable-lookahead-fsts" "--with-pic" "--disable-bin")))))
 
+(define-public sentencepiece
+  (package
+    (name "sentencepiece")
+    (version "0.1.97")
+    (source
+     (origin
+       (method git-fetch)
+       (uri (git-reference
+             (url "https://github.com/google/sentencepiece")
+             (commit (string-append "v" version))))
+       (file-name (git-file-name name version))
+       (sha256
+        (base32 "1kzfkp2pk0vabyw3wmkh16h11chzq63mzc20ddhsag5fp6s91ajg"))))
+    (build-system cmake-build-system)
+    (arguments '(#:tests? #f))
+    (native-inputs (list gperftools))
+    (home-page "https://github.com/google/sentencepiece")
+    (synopsis "Unsupervised tokenizer for Neural Network-based text generation")
+    (description "SentencePiece is an unsupervised text tokenizer and
+detokenizer mainly for Neural Network-based text generation systems where the
+vocabulary size is predetermined prior to the neural model training.
+SentencePiece implements subword units (e.g., byte-pair-encoding
+(BPE) and unigram language model) with the extension of direct training from
+raw sentences.  SentencePiece allows us to make a purely end-to-end system
+that does not depend on language-specific pre/postprocessing.")
+    (license license:asl2.0)))
+
 (define-public shogun
   (package
     (name "shogun")
-- 
2.39.2

Previous Next

GNU bug tracking system
Copyright (C) 1999 Darren O. Benham, 1997,2003 nCipher Corporation Ltd, 1994-97 Ian Jackson.