Add

cmd/job/job.go

@@ -11,8 +11,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/gocolly/colly"
 	"net/url"
-	"strconv"
 	"strings"
+	"html"
 )
 
 type TtsRaw struct {
@@ -32,6 +32,18 @@ type langType struct {
 	voiceName string;
 }
 
+
+type Phrase struct {
+	Text   			string
+	Paraphrase    	string
+	Type 			int64
+	Word			string
+}
+
+func (Phrase) TableName() string {
+	return "lnk_corpus_phrase_spider"
+}
+
 func RunCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "job",
@@ -52,23 +64,77 @@ func RunCommand() *cobra.Command {
 				panic("failed to connect database")
 			}
 
-			speed, _ := strconv.ParseFloat( "1.00", 64)
-			pitch, _ := strconv.ParseFloat( "0.00", 64)
-
 			db.Where("id <= ? and id >= ?", maxId, minId).Find(&ttsRawList)
 
 			for _, ttsRaw := range ttsRawList {
 
-				result, msg := SyncTtsOss(ttsRaw.Text, speed, pitch)
-
-				if result == true {
-					ttsRaw.Status = 1
+				word := ttsRaw.Text
+				c := colly.NewCollector(
+					// Visit only domains: hackerspaces.org, wiki.hackerspaces.org
+					colly.AllowedDomains("dict.cn", "m.dict.cn"),
+				)
+
+				// 例句
+				c.OnHTML("div[class=\"layout sort\"]", func(e *colly.HTMLElement) {
+					e.ForEach("ol", func(_ int, eol *colly.HTMLElement) {
+						eol.ForEach("li", func(_ int, el *colly.HTMLElement) {
+							//fmt.Println(el.DOM.Html())
+							liText, _ := el.DOM.Html()
+							liData := strings.Split(html.UnescapeString(liText), "<br/>")
+							example := ReplaceTrim(liData[0])
+							exampleParaphrase := ReplaceTrim(liData[1])
+
+							examplePhrase := Phrase{Text:example, Paraphrase:exampleParaphrase, Type: 0, Word:word}
+							fmt.Println(examplePhrase)
+							db.Create(&examplePhrase)
+						})
+					})
+				})
+
+				// 词汇搭配, 短语
+				c.OnHTML("div[class=\"layout coll\"]", func(e *colly.HTMLElement) {
+
+					e.ForEach("li", func(_ int, el *colly.HTMLElement) {
+						if el.ChildAttr("a", "href") != "" {
+							phrase := ReplaceTrim(el.ChildText("a"))
+							paraphrase := ReplaceTrim(el.Text)
+							paraphrase = strings.Replace(paraphrase, phrase, "", 1)
+
+							newPhrase := Phrase{Text:phrase, Paraphrase:paraphrase, Type: 1, Word:word}
+							fmt.Println(newPhrase)
+							db.Create(&newPhrase)
+						}
+					})
+				})
+
+				c.OnHTML("div[class=\"layout anno\"]", func(e *colly.HTMLElement) {
+
+					e.ForEach("li", func(_ int, el *colly.HTMLElement) {
+						if el.ChildAttr("a", "href") != "" {
+							phrase := ReplaceTrim(el.ChildText("a"))
+							paraphrase := ReplaceTrim(el.Text)
+							paraphrase = strings.Replace(paraphrase, phrase, "", 1)
+
+							newPhrase := Phrase{Text:phrase, Paraphrase:paraphrase, Type: 1, Word:word}
+							fmt.Println(newPhrase)
+							db.Create(&newPhrase)
+						}
+					})
+				})
+				c.OnRequest(func(r *colly.Request) {
+					fmt.Println("Visiting", r.URL.String())
+				})
+
+				c.Visit("http://dict.cn/"+word)
+
+				//if result == true {
+				ttsRaw.Status = 1
 					//更新成功
-				} else {
-					ttsRaw.Status = -1
-					//更新错误状态和msg
-					ttsRaw.Remark = msg
-				}
+				//} else {
+				//	ttsRaw.Status = -1
+				//	//更新错误状态和msg
+				//	ttsRaw.Remark = msg
+				//}
 				fmt.Println(ttsRaw)
 				db.Save(&ttsRaw)
 			}
@@ -128,49 +194,11 @@ func SyncTtsOss(text string, speed float64, pitch float64) (bool, string) {
 	return true, ""
 }
 
-func test() {
-
-	// Instantiate default collector
-	c := colly.NewCollector(
-		// Visit only domains: hackerspaces.org, wiki.hackerspaces.org
-		colly.AllowedDomains("dict.cn", "m.dict.cn"),
-	)
-
-	// On every a element which has href attribute call callback
-	c.OnHTML("div[class=sent]", func(e *colly.HTMLElement) {
-		//link := e.Attr("href")
-		// Print link
-		fmt.Printf("Link found: %q -> %s\n", e.Text)
-		// Visit link found on page
-		// Only those links are visited which are in AllowedDomains
-		//c.Visit(e.Request.AbsoluteURL(link))
-	})
-
-	/**
-	1. div[class=sent]
-	2. 例句 div[class=layout sort]
-	3. 词法用法 div[class=section learn]
-	4.
-	 */
-
-	// Before making a request print "Visiting ..."
-	c.OnRequest(func(r *colly.Request) {
-		fmt.Println("Visiting", r.URL.String())
-	})
-
-	// Start scraping on https://hackerspaces.org
-	c.Visit("http://dict.cn/about")
-}
+func ReplaceTrim(str string) string {
 
-func newTestCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "test",
-		Short: "Run the test service",
-		Run: func(cmd *cobra.Command, args []string) {
-			//fmt.Println("Echo: " + strings.Join(args, " "))
+	str = strings.Replace(str, "\n", "", -1)
+	str = strings.Replace(str, "\t", "", -1)
+
+	return strings.Trim(str, "")
+}
 
-			test()
-		},
-	}
-	return cmd
-}

main.go

@@ -29,7 +29,6 @@ func main() {
 	rootCmd.AddCommand(http.RunCommand())
 	rootCmd.AddCommand(version.RunCommand(apiVersion, gitCommit, built))
 	rootCmd.AddCommand(job.RunCommand())
-	rootCmd.AddCommand(job.newTestCmd())
 
 	if err := rootCmd.Execute(); err != nil {
 		panic(err)

service/store/mysql/tts_raw.go

@@ -0,0 +1,32 @@
+package mysql
+
+import (
+	"fmt"
+)
+
+const ttsRawCollection = `lnk_corpus_tts_raw`
+type TtsRaw struct {
+	ID                     int64  `db:"id"`
+	Text                   string  `db:"text"`			//翻译的文本
+	UniqKey                string  `db:"uniq_key"`		//唯一键值
+	Status				   int64  `db:"status"`
+	Remark                 string  `db:"remark"`
+}
+
+func GetTtsRawList() ([]TtsRaw, error)  {
+	var ttsRawList []TtsRaw
+
+	rows := db_session.Collection(ttsRawCollection).Find()
+	//if err != nil {
+	//
+	//	fmt.Println(err)
+	//	return ttsRawList, err
+	//}
+
+	err := rows.All(&ttsRawList)
+	if err != nil {
+	}
+	fmt.Println(ttsRawList) // SELECT id, name FROM accounts
+
+	return ttsRawList, err
+}