Initial commit: docker compose config

server/rutor/mem_test.go

@@ -0,0 +1,77 @@
+package rutor
+
+import (
+	"compress/flate"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"server/rutor/models"
+)
+
+func TestParseChannel(t *testing.T) {
+	channel := make(chan *models.TorrentDetails, 0)
+	var ftors []*models.TorrentDetails
+	go func() {
+		for torr := range channel {
+			ftors = append(ftors, torr)
+		}
+	}()
+
+	path, _ := os.Getwd()
+	ff, err := os.Open(filepath.Join(path, "rutor.ls"))
+	if err == nil {
+		defer ff.Close()
+		r := flate.NewReader(ff)
+		defer r.Close()
+		dec := json.NewDecoder(r)
+
+		_, err := dec.Token()
+		if err != nil {
+			t.Error(err)
+		}
+
+		for dec.More() {
+			var torr *models.TorrentDetails
+			err = dec.Decode(&torr)
+			if err != nil {
+				t.Error(err)
+			}
+			channel <- torr
+		}
+		close(channel)
+	} else {
+		t.Error(err)
+	}
+}
+
+func TestParseArr(t *testing.T) {
+	var ftors []*models.TorrentDetails
+	path, _ := os.Getwd()
+	ff, err := os.Open(filepath.Join(path, "rutor.ls"))
+	if err == nil {
+		defer ff.Close()
+		r := flate.NewReader(ff)
+		defer r.Close()
+		dec := json.NewDecoder(r)
+
+		_, err := dec.Token()
+		if err != nil {
+			t.Error(err)
+		}
+
+		for dec.More() {
+			var torr *models.TorrentDetails
+			err = dec.Decode(&torr)
+			if err != nil {
+				t.Error(err)
+			}
+			ftors = append(ftors, torr)
+			fmt.Println(len(ftors))
+		}
+	} else {
+		t.Error(err)
+	}
+}

server/rutor/models/torrentDetails.go

@@ -0,0 +1,76 @@
+package models
+
+import (
+	"strings"
+	"time"
+)
+
+const (
+	CatMovie         = "Movie"
+	CatSeries        = "Series"
+	CatDocMovie      = "DocMovie"
+	CatDocSeries     = "DocSeries"
+	CatCartoonMovie  = "CartoonMovie"
+	CatCartoonSeries = "CartoonSeries"
+	CatTVShow        = "TVShow"
+	CatAnime         = "Anime"
+
+	Q_LOWER           = 0
+	Q_WEBDL_720       = 100
+	Q_BDRIP_720       = 101
+	Q_BDRIP_HEVC_720  = 102
+	Q_WEBDL_1080      = 200
+	Q_BDRIP_1080      = 201
+	Q_BDRIP_HEVC_1080 = 202
+	Q_BDREMUX_1080    = 203
+	Q_WEBDL_SDR_2160  = 300
+	Q_WEBDL_HDR_2160  = 301
+	Q_WEBDL_DV_2160   = 302
+	Q_BDRIP_SDR_2160  = 303
+	Q_BDRIP_HDR_2160  = 304
+	Q_BDRIP_DV_2160   = 305
+	Q_UHD_BDREMUX_SDR = 306
+	Q_UHD_BDREMUX_HDR = 307
+	Q_UHD_BDREMUX_DV  = 308
+
+	Q_UNKNOWN = 0
+	Q_A       = 1   // Авторский, по типу Гоблина или старых переводчиков
+	Q_L1      = 100 // Любительский одноголосый закадровый
+	Q_L2      = 101 // Любительский двухголосый закадровый
+	Q_L       = 102 // Любительский 3-5 человек закадровый
+	Q_LS      = 103 // Любительский студия
+	Q_P1      = 200 // Професиональный одноголосый закадровый
+	Q_P2      = 201 // Профессиональный двухголосый закадровый
+	Q_P       = 202 // Профессиональный 3-5 человек закадровый
+	Q_PS      = 203 // Профессиональный студия
+	Q_D       = 300 // Официальное профессиональное многоголосое озвучивание
+	Q_LICENSE = 301 // Лицензия
+)
+
+type TorrentDetails struct {
+	Title        string
+	Name         string
+	Names        []string
+	Categories   string
+	Size         string
+	CreateDate   time.Time
+	Tracker      string
+	Link         string
+	Year         int
+	Peer         int
+	Seed         int
+	Magnet       string
+	Hash         string
+	IMDBID       string
+	VideoQuality int
+	AudioQuality int
+}
+
+type TorrentFile struct {
+	Name string
+	Size int64
+}
+
+func (d TorrentDetails) GetNames() string {
+	return strings.Join(d.Names, " ")
+}

server/rutor/race_test.go

@@ -0,0 +1,97 @@
+package rutor
+
+import (
+	"bytes"
+	"compress/flate"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"server/rutor/models"
+	"server/settings"
+)
+
+// TestConcurrentSearchAndLoadDB проверяет отсутствие гонки при одновременном
+// обновлении индекса (loadDB) и поиске (Search). 
+// !Запускать с -count=3
+func TestConcurrentSearchAndLoadDB(t *testing.T) {
+	if settings.BTsets == nil {
+		settings.BTsets = &settings.BTSets{EnableRutorSearch: true}
+		defer func() { settings.BTsets = nil }()
+	} else {
+		old := settings.BTsets.EnableRutorSearch
+		settings.BTsets.EnableRutorSearch = true
+		defer func() { settings.BTsets.EnableRutorSearch = old }()
+	}
+
+	dir := t.TempDir()
+	oldPath := settings.Path
+	settings.Path = dir
+	defer func() { settings.Path = oldPath }()
+
+	const numTorrents = 800
+	seed := make([]*models.TorrentDetails, numTorrents)
+	for i := 0; i < numTorrents; i++ {
+		s := strconv.Itoa(i)
+		seed[i] = &models.TorrentDetails{
+			Title: "Test Film Number " + s + " Part One Two Three Year",
+			Name:  "Film " + s,
+			Year:  2015 + i%10,
+		}
+	}
+	data, err := json.Marshal(seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var compressed bytes.Buffer
+	w, _ := flate.NewWriter(&compressed, flate.DefaultCompression)
+	_, _ = w.Write(data)
+	_ = w.Close()
+	if err := os.WriteFile(filepath.Join(dir, "rutor.ls"), compressed.Bytes(), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	done := make(chan struct{})
+	var wg sync.WaitGroup
+
+	// Горутина: многократно перезагружает БД (долгая перезапись индекса)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 20; i++ {
+			select {
+			case <-done:
+				return
+			default:
+				loadDB()
+				time.Sleep(5 * time.Millisecond)
+			}
+		}
+	}()
+
+	// Несколько горутин: постоянный поиск, пока идёт переиндексация
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			queries := []string{"Test", "Film", "Number", "Part", "Year", "xxx"}
+			for j := 0; j < 200; j++ {
+				select {
+				case <-done:
+					return
+				default:
+					_ = Search(queries[j%len(queries)])
+				}
+			}
+		}()
+	}
+
+	// Даём время на пересечение loadDB и Search
+	time.Sleep(800 * time.Millisecond)
+	close(done)
+	wg.Wait()
+}

server/rutor/rutor.go

@@ -0,0 +1,183 @@
+package rutor
+
+import (
+	"compress/flate"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/agnivade/levenshtein"
+
+	"server/log"
+	"server/rutor/models"
+	"server/rutor/torrsearch"
+	"server/rutor/utils"
+	"server/settings"
+	utils2 "server/torr/utils"
+)
+
+var (
+	mu     sync.RWMutex
+	torrs  []*models.TorrentDetails
+	isStop bool
+)
+
+func Start() {
+	go func() {
+		if settings.BTsets.EnableRutorSearch {
+			if !updateDB() {
+				loadDB()
+			}
+			isStop = false
+			for !isStop {
+				for i := 0; i < 3*60*60; i++ {
+					time.Sleep(time.Second)
+					if isStop {
+						return
+					}
+				}
+				updateDB()
+			}
+		}
+	}()
+}
+
+func Stop() {
+	mu.Lock()
+	isStop = true
+	torrs = nil
+	torrsearch.NewIndex(nil)
+	mu.Unlock()
+	utils2.FreeOSMemGC()
+	time.Sleep(time.Millisecond * 1500)
+}
+
+// http://releases.yourok.ru/torr/rutor.ls
+func updateDB() bool {
+	log.TLogln("Update rutor db")
+
+	fnOrig := filepath.Join(settings.Path, "rutor.ls")
+
+	if fi, err := os.Stat(fnOrig); err == nil {
+		if time.Since(fi.ModTime()) < time.Minute*175 /*2:55*/ {
+			log.TLogln("Less 3 hours rutor db old")
+			return false
+		}
+	}
+
+	fnTmp := filepath.Join(settings.Path, "rutor.tmp")
+	out, err := os.Create(fnTmp)
+	if err != nil {
+		log.TLogln("Error create file rutor.tmp:", err)
+		return false
+	}
+
+	resp, err := http.Get("http://releases.yourok.ru/torr/rutor.ls")
+	if err != nil {
+		log.TLogln("Error connect to rutor db:", err)
+		out.Close()
+		return false
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(out, resp.Body)
+	out.Close()
+	if err != nil {
+		log.TLogln("Error download rutor db:", err)
+		return false
+	}
+
+	md5Tmp := utils.MD5File(fnTmp)
+	md5Orig := utils.MD5File(fnOrig)
+	if md5Tmp != md5Orig {
+		err = os.Remove(fnOrig)
+		if err != nil && !os.IsNotExist(err) {
+			log.TLogln("Error remove old rutor db:", err)
+			return false
+		}
+		err = os.Rename(fnTmp, fnOrig)
+		if err != nil {
+			log.TLogln("Error rename rutor db:", err)
+			return false
+		}
+		loadDB()
+		return true
+	} else {
+		os.Remove(fnTmp)
+	}
+	return false
+}
+
+func loadDB() {
+	log.TLogln("Load rutor db")
+	ff, err := os.Open(filepath.Join(settings.Path, "rutor.ls"))
+	if err == nil {
+		defer ff.Close()
+		r := flate.NewReader(ff)
+		defer r.Close()
+		var ftorrs []*models.TorrentDetails
+		dec := json.NewDecoder(r)
+
+		_, err := dec.Token()
+		if err != nil {
+			log.TLogln("Error read token rutor db:", err)
+			return
+		}
+
+		for dec.More() {
+			var torr *models.TorrentDetails
+			err = dec.Decode(&torr)
+			if err == nil {
+				ftorrs = append(ftorrs, torr)
+			}
+		}
+		mu.Lock()
+		defer mu.Unlock()
+		torrs = ftorrs
+		log.TLogln("Index rutor db")
+		torrsearch.NewIndex(torrs)
+		log.TLogln("Torrents count:", len(torrs))
+		log.TLogln("Indexed words:", len(torrsearch.GetIDX()))
+
+	} else {
+		log.TLogln("Error load rutor db:", err)
+	}
+	utils2.FreeOSMemGC()
+}
+
+func Search(query string) []*models.TorrentDetails {
+	if !settings.BTsets.EnableRutorSearch {
+		return nil
+	}
+	mu.RLock()
+	matchedIDs := torrsearch.Search(query)
+	if len(matchedIDs) == 0 {
+		mu.RUnlock()
+		return nil
+	}
+	var list []*models.TorrentDetails
+	for _, id := range matchedIDs {
+		list = append(list, torrs[id])
+	}
+	mu.RUnlock()
+
+	hash := utils.ClearStr(query)
+
+	sort.Slice(list, func(i, j int) bool {
+		lhash := utils.ClearStr(strings.ToLower(list[i].Name+list[i].GetNames())) + strconv.Itoa(list[i].Year)
+		lev1 := levenshtein.ComputeDistance(hash, lhash)
+		lhash = utils.ClearStr(strings.ToLower(list[j].Name+list[j].GetNames())) + strconv.Itoa(list[j].Year)
+		lev2 := levenshtein.ComputeDistance(hash, lhash)
+		if lev1 == lev2 {
+			return list[j].CreateDate.Before(list[i].CreateDate)
+		}
+		return lev1 < lev2
+	})
+	return list
+}

server/rutor/torrsearch/filter.go

@@ -0,0 +1,68 @@
+package torrsearch
+
+import (
+	"strings"
+
+	snowballeng "github.com/kljensen/snowball/english"
+	snowballru "github.com/kljensen/snowball/russian"
+)
+
+// lowercaseFilter returns a slice of tokens normalized to lower case.
+func lowercaseFilter(tokens []string) []string {
+	r := make([]string, len(tokens))
+	for i, token := range tokens {
+		r[i] = replaceChars(strings.ToLower(token))
+	}
+	return r
+}
+
+// stopwordFilter returns a slice of tokens with stop words removed.
+func stopwordFilter(tokens []string) []string {
+	r := make([]string, 0, len(tokens))
+	for _, token := range tokens {
+		if !isStopWord(token) {
+			r = append(r, token)
+		}
+	}
+	return r
+}
+
+// stemmerFilter returns a slice of stemmed tokens.
+func stemmerFilter(tokens []string) []string {
+	r := make([]string, len(tokens))
+	for i, token := range tokens {
+		worden := snowballeng.Stem(token, false)
+		wordru := snowballru.Stem(token, false)
+		if wordru == "" || worden == "" {
+			continue
+		}
+		if wordru != token {
+			r[i] = wordru
+		} else {
+			r[i] = worden
+		}
+	}
+	return r
+}
+
+func replaceChars(word string) string {
+	out := []rune(word)
+	for i, r := range out {
+		if r == 'ё' {
+			out[i] = 'е'
+		}
+	}
+	return string(out)
+}
+
+func isStopWord(word string) bool {
+	switch word {
+	case "a", "am", "an", "and", "are", "as", "at", "be",
+		"by", "did", "do", "is", "of", "or", "s", "so", "t",
+		"и", "в", "с", "со", "а", "но", "к", "у",
+		"же", "бы", "по", "от", "о", "из", "ну",
+		"ли", "ни", "нибудь", "уж", "ведь", "ж", "об":
+		return true
+	}
+	return false
+}

server/rutor/torrsearch/index.go

@@ -0,0 +1,77 @@
+package torrsearch
+
+import (
+	"server/rutor/models"
+)
+
+// Index is an inverted Index. It maps tokens to document IDs.
+type Index map[string][]int
+
+var idx Index
+
+func NewIndex(torrs []*models.TorrentDetails) {
+	idx = make(Index)
+	idx.add(torrs)
+}
+
+func Search(text string) []int {
+	return idx.search(text)
+}
+
+func GetIDX() Index {
+	return idx
+}
+
+func (idx Index) add(torrs []*models.TorrentDetails) {
+	for ID, torr := range torrs {
+		for _, token := range analyze(torr.Title) {
+			ids := idx[token]
+			if ids != nil && ids[len(ids)-1] == ID {
+				// Don't add same ID twice.
+				continue
+			}
+			idx[token] = append(ids, ID)
+		}
+	}
+}
+
+// intersection returns the set intersection between a and b.
+// a and b have to be sorted in ascending order and contain no duplicates.
+func intersection(a []int, b []int) []int {
+	maxLen := len(a)
+	if len(b) > maxLen {
+		maxLen = len(b)
+	}
+	r := make([]int, 0, maxLen)
+	var i, j int
+	for i < len(a) && j < len(b) {
+		if a[i] < b[j] {
+			i++
+		} else if a[i] > b[j] {
+			j++
+		} else {
+			r = append(r, a[i])
+			i++
+			j++
+		}
+	}
+	return r
+}
+
+// Search queries the Index for the given text.
+func (idx Index) search(text string) []int {
+	var r []int
+	for _, token := range analyze(text) {
+		if ids, ok := idx[token]; ok {
+			if r == nil {
+				r = ids
+			} else {
+				r = intersection(r, ids)
+			}
+		} else {
+			// Token doesn't exist.
+			return nil
+		}
+	}
+	return r
+}

server/rutor/torrsearch/tokenizer.go

@@ -0,0 +1,23 @@
+package torrsearch
+
+import (
+	"strings"
+	"unicode"
+)
+
+// tokenize returns a slice of tokens for the given text.
+func tokenize(text string) []string {
+	return strings.FieldsFunc(text, func(r rune) bool {
+		// Split on any character that is not a letter or a number.
+		return !unicode.IsLetter(r) && !unicode.IsNumber(r)
+	})
+}
+
+// analyze analyzes the text and returns a slice of tokens.
+func analyze(text string) []string {
+	tokens := tokenize(text)
+	tokens = lowercaseFilter(tokens)
+	tokens = stopwordFilter(tokens)
+	// tokens = stemmerFilter(tokens)
+	return tokens
+}

server/rutor/utils/utils.go

@@ -0,0 +1,42 @@
+package utils
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"os"
+	"strings"
+)
+
+func ClearStr(str string) string {
+	ret := ""
+	str = strings.ToLower(str)
+	for _, r := range str {
+		if (r >= '0' && r <= '9') || (r >= 'a' && r <= 'z') || (r >= 'а' && r <= 'я') || r == 'ё' {
+			ret = ret + string(r)
+		}
+	}
+	return ret
+}
+
+func MD5File(fname string) string {
+	f, err := os.Open(fname)
+	if err != nil {
+		return ""
+	}
+
+	defer f.Close()
+
+	buf := make([]byte, 1024*1024)
+	h := sha256.New()
+
+	for {
+		bytesRead, err := f.Read(buf)
+		if err != nil {
+			break
+		}
+
+		h.Write(buf[:bytesRead])
+	}
+
+	return hex.EncodeToString(h.Sum(nil))
+}