Initial commit: docker compose config

server/rutor/torrsearch/tokenizer.go

@@ -0,0 +1,23 @@
+package torrsearch
+
+import (
+	"strings"
+	"unicode"
+)
+
+// tokenize returns a slice of tokens for the given text.
+func tokenize(text string) []string {
+	return strings.FieldsFunc(text, func(r rune) bool {
+		// Split on any character that is not a letter or a number.
+		return !unicode.IsLetter(r) && !unicode.IsNumber(r)
+	})
+}
+
+// analyze analyzes the text and returns a slice of tokens.
+func analyze(text string) []string {
+	tokens := tokenize(text)
+	tokens = lowercaseFilter(tokens)
+	tokens = stopwordFilter(tokens)
+	// tokens = stemmerFilter(tokens)
+	return tokens
+}