moredoc/util/segword/jieba/jieba.go

package jieba

import (
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/wangbin/jiebago"
)

var (
	chineseChar = map[string]bool{
		"，": true, "。": true, "？": true, "！": true, "、": true, "：": true, "；": true,
		"（": true, "）": true, "《": true, "》": true, "“": true, "”": true, "‘": true,
		"’": true, "【": true, "】": true, "『": true, "』": true, "〔": true, "〕": true,
		"〈": true, "〉": true, "﹑": true, "●": true, "…": true, "—": true, "～": true,
	}
	seg        jiebago.Segmenter
	dictPath   = "dictionary/dict.txt"
	dictLoaded = false
)

func init() {
	err := loadDictionary()
	if err != nil {
		panic(err)
	}
}

func loadDictionary(dict ...string) error {
	if dictLoaded {
		return nil
	}
	if len(dict) > 0 {
		path := dict[0]
		if path != "" {
			dictPath = path
		}
	}
	err := seg.LoadDictionary(dictPath)
	if err == nil {
		dictLoaded = true
	}
	return err
}

func SegWords(text string) []string {
	words := make([]string, 0)
	exist := make(map[string]bool)
	for word := range seg.CutForSearch(text, true) {
		wd := strings.TrimSpace(word)
		if _, ok := exist[wd]; ok {
			continue
		}
		exist[wd] = true
		l := utf8.RuneCountInString(wd)
		if !(l < 2 || (l == 1 && (isChineseChar(wd) || unicode.IsPunct(rune(wd[0]))))) { // 跳过空格和标点符号
			words = append(words, wd)
		}
	}
	return words
}

func isChineseChar(term string) bool {
	_, ok := chineseChar[term]
	return ok
}